[llvm] [AMDGPU] Add GFX1250 unclaused VMEM workaround (PR #208467)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 09:28:34 PDT 2026
https://github.com/jayfoad updated https://github.com/llvm/llvm-project/pull/208467
>From e21919af76c8b36cd9dd4b699869712d3a72980d Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Fri, 12 Jun 2026 11:04:57 +0100
Subject: [PATCH 1/2] [AMDGPU] Add GFX1250 unclaused VMEM workaround
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 6 +
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 13 +
.../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 96 ++++
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 22 +
.../CodeGen/AMDGPU/GlobalISel/llvm.abs.ll | 16 +
...llvm.amdgcn.raw.ptr.buffer.offset-split.ll | 6 +
.../AMDGPU/GlobalISel/load-constant.96.ll | 20 +
.../AMDGPU/GlobalISel/minmaxabs-i64.ll | 10 +
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 30 ++
llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll | 4 +
llvm/test/CodeGen/AMDGPU/add-max.ll | 44 ++
llvm/test/CodeGen/AMDGPU/add_u64.ll | 16 +
llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll | 14 +
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 8 +
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 28 +
llvm/test/CodeGen/AMDGPU/bf16-math.ll | 124 +++++
llvm/test/CodeGen/AMDGPU/bf16.ll | 26 +
llvm/test/CodeGen/AMDGPU/bitop3.ll | 96 ++++
.../AMDGPU/branch-relaxation-gfx1250.ll | 44 ++
.../CodeGen/AMDGPU/calling-conventions.ll | 110 ++++
.../test/CodeGen/AMDGPU/carryout-selection.ll | 34 ++
llvm/test/CodeGen/AMDGPU/clmul.ll | 6 +
.../test/CodeGen/AMDGPU/code-size-estimate.ll | 4 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 8 +
llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll | 56 ++
llvm/test/CodeGen/AMDGPU/ds_write2.ll | 46 ++
.../test/CodeGen/AMDGPU/fcanonicalize.bf16.ll | 138 +++++
llvm/test/CodeGen/AMDGPU/fcanonicalize.ll | 366 +++++++++++++
.../CodeGen/AMDGPU/fence-barrier-latency.ll | 2 +
.../AMDGPU/flat-load-saddr-to-vaddr.ll | 2 +
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 332 ++++++++++++
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 464 ++++++++++++++++
llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll | 214 ++++++++
llvm/test/CodeGen/AMDGPU/fmax3.ll | 12 +
llvm/test/CodeGen/AMDGPU/fmin3.ll | 16 +
llvm/test/CodeGen/AMDGPU/fp-atomics-gfx942.ll | 8 +
.../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 98 ++++
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 104 ++++
llvm/test/CodeGen/AMDGPU/global-address.ll | 8 +
llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll | 4 +
.../CodeGen/AMDGPU/insert-delay-alu-attr.ll | 8 +
.../AMDGPU/insert_vector_elt.v2bf16.ll | 56 ++
.../AMDGPU/intrinsic-amdgcn-s-alloc-vgpr.ll | 18 +
llvm/test/CodeGen/AMDGPU/literal64.ll | 38 ++
.../AMDGPU/llvm.amdgcn.av.store.b128.ll | 8 +
.../test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll | 54 ++
.../CodeGen/AMDGPU/llvm.amdgcn.cluster.id.ll | 24 +
.../llvm.amdgcn.cluster.load.async.to.lds.ll | 50 ++
.../AMDGPU/llvm.amdgcn.cluster.load.ll | 28 +
.../llvm.amdgcn.cluster.workgroup.id.ll | 56 ++
...vm.amdgcn.cluster.workgroup.max.flat.id.ll | 8 +
.../llvm.amdgcn.cluster.workgroup.max.id.ll | 48 ++
.../CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll | 12 +
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll | 124 +++++
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll | 22 +
.../AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll | 22 +
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll | 132 +++++
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll | 40 ++
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.f16.ll | 10 +
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll | 48 ++
.../AMDGPU/llvm.amdgcn.cvt.scale.pk.ll | 98 ++++
.../llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll | 64 +++
.../AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll | 96 ++++
.../llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll | 96 ++++
.../llvm.amdgcn.cvt.scalef32.sr.pk16.ll | 48 ++
.../AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll | 10 +
...dgcn.ds.atomic.async.barrier.arrive.b64.ll | 2 +
...amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll | 4 +
.../CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll | 12 +
.../AMDGPU/llvm.amdgcn.flat.prefetch.ll | 20 +
.../llvm.amdgcn.global.load.async.to.lds.ll | 30 ++
.../AMDGPU/llvm.amdgcn.global.prefetch.ll | 20 +
...llvm.amdgcn.global.store.async.from.lds.ll | 30 ++
.../CodeGen/AMDGPU/llvm.amdgcn.is.private.ll | 6 +
.../CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll | 6 +
.../llvm.amdgcn.load.monitor.gfx1250.ll | 28 +
.../AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll | 54 ++
.../CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll | 12 +
.../CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll | 12 +
.../llvm.amdgcn.raw.atomic.buffer.load.ll | 32 ++
.../AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll | 20 +
.../AMDGPU/llvm.amdgcn.raw.buffer.store.ll | 50 ++
.../llvm.amdgcn.raw.ptr.atomic.buffer.load.ll | 32 ++
.../AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll | 56 ++
.../llvm.amdgcn.raw.ptr.buffer.store.bf16.ll | 8 +
.../CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll | 16 +
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll | 16 +
.../AMDGPU/llvm.amdgcn.s.buffer.load.ll | 74 +++
.../AMDGPU/llvm.amdgcn.s.prefetch.inst.ll | 62 +++
.../AMDGPU/llvm.amdgcn.s.wait.gfx1250.ll | 4 +
.../llvm.amdgcn.sched.group.barrier.gfx12.ll | 4 +
.../CodeGen/AMDGPU/llvm.amdgcn.sendmsg.rtn.ll | 26 +
.../CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll | 12 +
.../CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll | 12 +
.../llvm.amdgcn.struct.atomic.buffer.load.ll | 34 ++
.../llvm.amdgcn.struct.buffer.load.tfe.ll | 20 +
.../AMDGPU/llvm.amdgcn.struct.buffer.store.ll | 36 ++
...vm.amdgcn.struct.ptr.atomic.buffer.load.ll | 34 ++
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll | 36 ++
.../AMDGPU/llvm.amdgcn.tensor.load.store.ll | 34 ++
.../CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll | 82 +++
.../CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll | 2 +
.../AMDGPU/llvm.amdgcn.wmma.gfx1250.w32.ll | 500 ++++++++++++++++++
.../AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll | 8 +
.../llvm.amdgcn.wmma.imm.gfx1250.w32.ll | 348 ++++++++++++
.../llvm.amdgcn.wmma.imm.gfx1251.w32.ll | 12 +
.../llvm.amdgcn.wmma.imod.gfx1250.w32.ll | 392 ++++++++++++++
.../llvm.amdgcn.wmma.imod.gfx1251.w32.ll | 20 +
.../llvm.amdgcn.wmma.index.gfx1250.w32.ll | 128 +++++
llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll | 12 +
llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll | 134 +++++
llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll | 12 +
llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll | 8 +
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 112 ++++
llvm/test/CodeGen/AMDGPU/load-constant-i32.ll | 50 ++
.../CodeGen/AMDGPU/load-saddr-offset-imm.ll | 6 +
.../test/CodeGen/AMDGPU/loop-prefetch-data.ll | 12 +
.../AMDGPU/lower-work-group-id-intrinsics.ll | 16 +
llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll | 2 +
llvm/test/CodeGen/AMDGPU/mad_64_32.ll | 4 +
llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll | 28 +
llvm/test/CodeGen/AMDGPU/max.ll | 46 ++
.../AMDGPU/memory-legalizer-barriers.ll | 6 +
.../memory-legalizer-fence-mmra-global.ll | 48 ++
.../memory-legalizer-fence-mmra-local.ll | 48 ++
.../CodeGen/AMDGPU/memory-legalizer-fence.ll | 96 ++++
.../AMDGPU/memory-legalizer-flat-agent.ll | 184 +++++++
.../AMDGPU/memory-legalizer-flat-cluster.ll | 184 +++++++
.../AMDGPU/memory-legalizer-flat-lastuse.ll | 8 +
.../memory-legalizer-flat-nontemporal.ll | 10 +
.../memory-legalizer-flat-singlethread.ll | 184 +++++++
.../AMDGPU/memory-legalizer-flat-system.ll | 184 +++++++
.../AMDGPU/memory-legalizer-flat-volatile.ll | 12 +
.../AMDGPU/memory-legalizer-flat-wavefront.ll | 182 +++++++
.../AMDGPU/memory-legalizer-flat-workgroup.ll | 176 ++++++
.../AMDGPU/memory-legalizer-global-agent.ll | 182 +++++++
.../AMDGPU/memory-legalizer-global-cluster.ll | 182 +++++++
.../AMDGPU/memory-legalizer-global-lastuse.ll | 8 +
.../memory-legalizer-global-nontemporal.ll | 10 +
.../memory-legalizer-global-singlethread.ll | 184 +++++++
.../AMDGPU/memory-legalizer-global-system.ll | 174 ++++++
.../memory-legalizer-global-volatile.ll | 12 +
.../memory-legalizer-global-wavefront.ll | 184 +++++++
.../memory-legalizer-global-workgroup.ll | 184 +++++++
.../AMDGPU/memory-legalizer-local-agent.ll | 184 +++++++
.../AMDGPU/memory-legalizer-local-cluster.ll | 184 +++++++
.../memory-legalizer-local-nontemporal.ll | 10 +
.../memory-legalizer-local-singlethread.ll | 184 +++++++
.../AMDGPU/memory-legalizer-local-system.ll | 184 +++++++
.../AMDGPU/memory-legalizer-local-volatile.ll | 12 +
.../memory-legalizer-local-wavefront.ll | 184 +++++++
.../memory-legalizer-local-workgroup.ll | 184 +++++++
.../AMDGPU/memory-legalizer-private-agent.ll | 182 +++++++
.../memory-legalizer-private-cluster.ll | 182 +++++++
.../memory-legalizer-private-lastuse.ll | 8 +
.../memory-legalizer-private-nontemporal.ll | 10 +
.../memory-legalizer-private-singlethread.ll | 184 +++++++
.../AMDGPU/memory-legalizer-private-system.ll | 174 ++++++
.../memory-legalizer-private-volatile.ll | 8 +
.../memory-legalizer-private-wavefront.ll | 184 +++++++
.../memory-legalizer-private-workgroup.ll | 184 +++++++
llvm/test/CodeGen/AMDGPU/min.ll | 74 +++
llvm/test/CodeGen/AMDGPU/minmax.ll | 40 ++
llvm/test/CodeGen/AMDGPU/mul.ll | 38 ++
llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 156 ++++++
llvm/test/CodeGen/AMDGPU/packed-fp64.ll | 134 +++++
llvm/test/CodeGen/AMDGPU/packed-u64.ll | 70 +++
llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll | 12 +
.../AMDGPU/preload-implicit-kernargs.ll | 48 ++
llvm/test/CodeGen/AMDGPU/preload-kernargs.ll | 68 +++
.../promote-constOffset-to-imm-gfx12.ll | 6 +
.../AMDGPU/reassoc-mul-add-1-to-mad.ll | 6 +
.../AMDGPU/s-barrier-signal-var-gep.ll | 26 +
llvm/test/CodeGen/AMDGPU/s-cluster-barrier.ll | 2 +
llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll | 4 +
llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll | 48 ++
.../CodeGen/AMDGPU/scale-offset-global.ll | 46 ++
.../CodeGen/AMDGPU/scale-offset-scratch.ll | 44 ++
llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll | 42 ++
llvm/test/CodeGen/AMDGPU/shl.v2i64.ll | 36 ++
llvm/test/CodeGen/AMDGPU/spillv16Kernel.ll | 4 +
llvm/test/CodeGen/AMDGPU/sub_u64.ll | 18 +
llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll | 8 +
.../wait-before-stores-with-scope_sys.ll | 8 +
.../wait-xcnt-atomic-rmw-optimization.ll | 40 ++
llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir | 34 ++
llvm/test/CodeGen/AMDGPU/wait-xcnt.mir | 74 ++-
.../waitcnt-loop-ds-prefetch-flushed.ll | 2 +
.../waitcnt-loop-ds-prefetch-flushed.mir | 2 +
.../waitcnt-loop-ds-prefetch-pattern.ll | 2 +
.../AMDGPU/waitcnt-loop-ds-store-barrier.mir | 74 ++-
llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir | 4 +
.../CodeGen/AMDGPU/whole-wave-functions.ll | 2 +
.../AMDGPU/workgroup-id-in-arch-sgprs.ll | 12 +
194 files changed, 12502 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index a1298e5969ea0..8aed06ef918e4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1215,6 +1215,11 @@ defm FlatSignedOffset : AMDGPUSubtargetFeature<"flat-signed-offset",
"Immediate offset of FLAT instructions are always signed"
>;
+defm RequiresInitialUnclausedVmem : AMDGPUSubtargetFeature<"requires-initial-unclause-vmem",
+ "Hardware entrypoints require first VMEM instruction to be unclaused",
+ GenPredicate=false
+>;
+
//===------------------------------------------------------------===//
// Subtarget Features (options and debugging)
//===------------------------------------------------------------===//
@@ -2233,6 +2238,7 @@ def FeatureISAVersion12_50_Common : FeatureSet<
FeatureRealTrue16Insts,
FeatureVMovB64Inst,
FeatureVMulU64Inst,
+ FeatureRequiresInitialUnclausedVmem,
]>;
def FeatureISAVersion12_50 : FeatureSet<
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index c63c7c6345aa2..eb076b9a84c20 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -3629,5 +3629,18 @@ bool SIInsertWaitcnts::run() {
}
}
+ if (MFI->isEntryFunction() && ST.hasRequiresInitialUnclausedVmem()) {
+ // Hardware entrypoints must begin with a specific sequence:
+ // GLOBAL_WB SCOPE:SCOPE_CU
+ // V_NOP
+ MachineBasicBlock::iterator I = EntryBB.begin();
+ BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::GLOBAL_WB))
+ .addImm(AMDGPU::CPol::SCOPE_CU)
+ .addReg(AMDGPU::EXEC, RegState::Implicit);
+ BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::V_NOP_e32))
+ .addReg(AMDGPU::EXEC, RegState::Implicit);
+ Modified = true;
+ }
+
return Modified;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index e63c31f712725..5e7d20560706d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -41,6 +41,8 @@ define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -72,6 +74,8 @@ define amdgpu_ps void @raw_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -116,6 +120,8 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -163,6 +169,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -194,6 +202,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -238,6 +248,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -285,6 +297,8 @@ define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -316,6 +330,8 @@ define amdgpu_ps void @struct_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -360,6 +376,8 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -406,6 +424,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -437,6 +457,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -481,6 +503,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -527,6 +551,8 @@ define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -558,6 +584,8 @@ define amdgpu_ps void @raw_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -602,6 +630,8 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -649,6 +679,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -680,6 +712,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -724,6 +758,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -771,6 +807,8 @@ define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -802,6 +840,8 @@ define amdgpu_ps void @struct_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -846,6 +886,8 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -892,6 +934,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -923,6 +967,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -967,6 +1013,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1013,6 +1061,8 @@ define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1044,6 +1094,8 @@ define amdgpu_ps void @raw_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1088,6 +1140,8 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1135,6 +1189,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1166,6 +1222,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1210,6 +1268,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1257,6 +1317,8 @@ define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1288,6 +1350,8 @@ define amdgpu_ps void @struct_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1332,6 +1396,8 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1378,6 +1444,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1409,6 +1477,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1453,6 +1523,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1522,6 +1594,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -1596,6 +1670,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -1672,6 +1748,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -1746,6 +1824,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -1946,6 +2026,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -2005,6 +2087,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2050,6 +2134,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent(ptr %ptr) #1 {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2098,6 +2184,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_system:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2271,6 +2359,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2331,6 +2421,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
;
; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -2397,6 +2489,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
;
; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -2463,6 +2557,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
;
; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index f8c1fc2416cd8..28d7052b55470 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -26,6 +26,8 @@ define amdgpu_ps half @fptrunc_f32_to_f16_uniform(float inreg %a) {
;
; GFX1250-LABEL: fptrunc_f32_to_f16_uniform:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_cvt_f16_f32 s0, s0
@@ -59,6 +61,8 @@ define amdgpu_ps half @fptrunc_f32_to_f16_div(float %a) {
;
; GFX1250-FAKE16-LABEL: fptrunc_f32_to_f16_div:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -66,6 +70,8 @@ define amdgpu_ps half @fptrunc_f32_to_f16_div(float %a) {
;
; GFX1250-TRUE16-LABEL: fptrunc_f32_to_f16_div:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
@@ -87,6 +93,8 @@ define amdgpu_ps float @fptrunc_f64_to_f32_uniform(double inreg %a) {
;
; GFX1250-LABEL: fptrunc_f64_to_f32_uniform:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e32 v0, s[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -107,6 +115,8 @@ define amdgpu_ps float @fptrunc_f64_to_f32_div(double %a) {
;
; GFX1250-LABEL: fptrunc_f64_to_f32_div:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -221,6 +231,8 @@ define amdgpu_ps half @fptrunc_f64_to_f16_uniform(double inreg %a) {
;
; GFX1250-LABEL: fptrunc_f64_to_f16_uniform:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_bfe_u32 s2, s1, 0xb0014
; GFX1250-NEXT: s_lshr_b32 s3, s1, 8
@@ -379,6 +391,8 @@ define amdgpu_ps half @fptrunc_f64_to_f16_div(double %a) {
;
; GFX1250-LABEL: fptrunc_f64_to_f16_div:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_and_or_b32 v0, 0x1ff, v1, v0
; GFX1250-NEXT: v_bfe_u32 v2, v1, 20, 11
@@ -464,6 +478,8 @@ define amdgpu_ps <2 x half> @fptrunc_v2f32_to_v2f16_uniform(<2 x float> inreg %a
;
; GFX1250-LABEL: fptrunc_v2f32_to_v2f16_uniform:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -504,6 +520,8 @@ define amdgpu_ps <2 x half> @fptrunc_v2f32_to_v2f16_div(<2 x float> %a) {
;
; GFX1250-LABEL: fptrunc_v2f32_to_v2f16_div:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
@@ -528,6 +546,8 @@ define amdgpu_ps void @fptrunc_v2f64_to_v2f32_uniform(<2 x double> inreg %a, ptr
;
; GFX1250-LABEL: fptrunc_v2f64_to_v2f32_uniform:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
; GFX1250-NEXT: v_cvt_f32_f64_e32 v3, s[2:3]
@@ -555,6 +575,8 @@ define amdgpu_ps void @fptrunc_v2f64_to_v2f32_div(<2 x double> %a, ptr addrspace
;
; GFX1250-LABEL: fptrunc_v2f64_to_v2f32_div:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1250-NEXT: v_cvt_f32_f64_e32 v1, v[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
index e2e7e28b35564..0f31d60f4d96b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
@@ -23,6 +23,8 @@ define amdgpu_cs i16 @abs_sgpr_i16(i16 inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sext_i32_i16 s0, s0
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -40,6 +42,8 @@ define amdgpu_cs i32 @abs_sgpr_i32(i32 inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_abs_i32 s0, s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -59,6 +63,8 @@ define amdgpu_cs i64 @abs_sgpr_i64(i64 inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_ashr_i32 s2, s1, 31
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -82,6 +88,8 @@ define amdgpu_cs <4 x i32> @abs_sgpr_v4i32(<4 x i32> inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_v4i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_abs_i32 s0, s0
; GFX1250-NEXT: s_abs_i32 s1, s1
@@ -285,6 +293,8 @@ define amdgpu_cs <2 x i8> @abs_sgpr_v2i8(<2 x i8> inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_v2i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sext_i32_i8 s0, s0
; GFX1250-NEXT: s_sext_i32_i8 s1, s1
@@ -372,6 +382,8 @@ define amdgpu_cs <3 x i8> @abs_sgpr_v3i8(<3 x i8> inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_v3i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sext_i32_i8 s0, s0
; GFX1250-NEXT: s_sext_i32_i8 s1, s1
@@ -497,6 +509,8 @@ define amdgpu_cs <2 x i16> @abs_sgpr_v2i16(<2 x i16> inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sext_i32_i16 s1, s0
; GFX1250-NEXT: s_ashr_i32 s0, s0, 16
@@ -592,6 +606,8 @@ define amdgpu_cs <3 x i16> @abs_sgpr_v3i16(<3 x i16> inreg %arg) {
;
; GFX1250-LABEL: abs_sgpr_v3i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sext_i32_i16 s2, s0
; GFX1250-NEXT: s_ashr_i32 s0, s0, 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
index 55561c489402a..d905fa73c3a88 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll
@@ -16,6 +16,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
;
; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0x3800000
; GFX1250-NEXT: s_mov_b32 s0, s2
@@ -44,6 +46,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soff
;
; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__voffset67108860_soffset_neg16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0x3800000
; GFX1250-NEXT: s_mov_b32 s0, s2
@@ -73,6 +77,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or6710
;
; GFX1250-LABEL: raw_ptr_buffer_load_f32__sgpr_rsrc__vgpr_voffset__or67108860_soffset16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 0x3fffffc
; GFX1250-NEXT: s_mov_b32 s0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index d98ae47a6fd5e..793e4d8071d16 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -921,6 +921,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align1(ptr addrspace(4) inreg
;
; GFX1250-UNALIGNED-LABEL: s_load_constant_v3i32_align1:
; GFX1250-UNALIGNED: ; %bb.0:
+; GFX1250-UNALIGNED-NEXT: global_wb
+; GFX1250-UNALIGNED-NEXT: v_nop
; GFX1250-UNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v0, s[0:1] nv
@@ -932,6 +934,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align1(ptr addrspace(4) inreg
;
; GFX1250-NOUNALIGNED-LABEL: s_load_constant_v3i32_align1:
; GFX1250-NOUNALIGNED: ; %bb.0:
+; GFX1250-NOUNALIGNED-NEXT: global_wb
+; GFX1250-NOUNALIGNED-NEXT: v_nop
; GFX1250-NOUNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOUNALIGNED-NEXT: s_clause 0xb
; GFX1250-NOUNALIGNED-NEXT: s_load_u8 s2, s[0:1], 0x1 nv
@@ -1201,6 +1205,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align2(ptr addrspace(4) inreg
;
; GFX1250-UNALIGNED-LABEL: s_load_constant_v3i32_align2:
; GFX1250-UNALIGNED: ; %bb.0:
+; GFX1250-UNALIGNED-NEXT: global_wb
+; GFX1250-UNALIGNED-NEXT: v_nop
; GFX1250-UNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-UNALIGNED-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-UNALIGNED-NEXT: global_load_b96 v[0:2], v0, s[0:1] nv
@@ -1212,6 +1218,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align2(ptr addrspace(4) inreg
;
; GFX1250-NOUNALIGNED-LABEL: s_load_constant_v3i32_align2:
; GFX1250-NOUNALIGNED: ; %bb.0:
+; GFX1250-NOUNALIGNED-NEXT: global_wb
+; GFX1250-NOUNALIGNED-NEXT: v_nop
; GFX1250-NOUNALIGNED-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOUNALIGNED-NEXT: s_clause 0x5
; GFX1250-NOUNALIGNED-NEXT: s_load_u16 s2, s[0:1], 0x2 nv
@@ -1357,6 +1365,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align4(ptr addrspace(4) inreg
;
; GFX1250-LABEL: s_load_constant_v3i32_align4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s4, s0
; GFX1250-NEXT: s_mov_b32 s5, s1
@@ -1409,6 +1419,8 @@ define amdgpu_ps i96 @s_load_constant_i96_align8(ptr addrspace(4) inreg %ptr) {
;
; GFX1250-LABEL: s_load_constant_i96_align8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s4, s0
; GFX1250-NEXT: s_mov_b32 s5, s1
@@ -1461,6 +1473,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align8(ptr addrspace(4) inreg
;
; GFX1250-LABEL: s_load_constant_v3i32_align8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s4, s0
; GFX1250-NEXT: s_mov_b32 s5, s1
@@ -1513,6 +1527,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v6i16_align8(ptr addrspace(4) inreg
;
; GFX1250-LABEL: s_load_constant_v6i16_align8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s4, s0
; GFX1250-NEXT: s_mov_b32 s5, s1
@@ -1592,6 +1608,8 @@ define amdgpu_ps <12 x i8> @s_load_constant_v12i8_align8(ptr addrspace(4) inreg
;
; GFX1250-LABEL: s_load_constant_v12i8_align8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s4, s0
; GFX1250-NEXT: s_mov_b32 s5, s1
@@ -1684,6 +1702,8 @@ define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align16(ptr addrspace(4) inreg
;
; GFX1250-LABEL: s_load_constant_v3i32_align16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[0:1], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
index 6a7725d19086a..cdf85fc4de9c2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll
@@ -132,6 +132,8 @@ define i64 @test_abs_i64(i64 %a) {
define amdgpu_ps i64 @test_umin_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_umin_i64_s:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_min_u64 v[0:1], s[0:1], s[2:3]
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -145,6 +147,8 @@ define amdgpu_ps i64 @test_umin_i64_s(i64 inreg %a, i64 inreg %b) {
define amdgpu_ps i64 @test_umax_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_umax_i64_s:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_max_u64 v[0:1], s[0:1], s[2:3]
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -158,6 +162,8 @@ define amdgpu_ps i64 @test_umax_i64_s(i64 inreg %a, i64 inreg %b) {
define amdgpu_ps i64 @test_smin_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_smin_i64_s:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_min_i64 v[0:1], s[0:1], s[2:3]
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -171,6 +177,8 @@ define amdgpu_ps i64 @test_smin_i64_s(i64 inreg %a, i64 inreg %b) {
define amdgpu_ps i64 @test_smax_i64_s(i64 inreg %a, i64 inreg %b) {
; CHECK-LABEL: test_smax_i64_s:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_max_i64 v[0:1], s[0:1], s[2:3]
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -184,6 +192,8 @@ define amdgpu_ps i64 @test_smax_i64_s(i64 inreg %a, i64 inreg %b) {
define amdgpu_ps i64 @test_abs_i64_s(i64 inreg %a) {
; CHECK-LABEL: test_abs_i64_s:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_ashr_i32 s2, s1, 31
; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index e1dda7317c9fe..d6655fc0ab73f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -27,6 +27,8 @@ define amdgpu_ps i16 @s_mul_i16(i16 inreg %num, i16 inreg %den) {
;
; GFX1250-LABEL: s_mul_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
@@ -142,6 +144,8 @@ define amdgpu_ps zeroext i16 @s_mul_i16_zeroext(i16 inreg zeroext %num, i16 inre
;
; GFX1250-LABEL: s_mul_i16_zeroext:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s0, s0, s1
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -257,6 +261,8 @@ define amdgpu_ps signext i16 @s_mul_i16_signext(i16 inreg signext %num, i16 inre
;
; GFX1250-LABEL: s_mul_i16_signext:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s0, s0, s1
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -372,6 +378,8 @@ define amdgpu_ps i32 @s_mul_i32(i32 inreg %num, i32 inreg %den) {
;
; GFX1250-LABEL: s_mul_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
@@ -493,6 +501,8 @@ define amdgpu_ps <2 x i16> @s_mul_v2i16(<2 x i16> inreg %num, <2 x i16> inreg %d
;
; GFX1250-LABEL: s_mul_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_mul_lo_u16 v0, s0, s1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
@@ -605,6 +615,8 @@ define amdgpu_ps <2 x i32> @s_mul_v2i32(<2 x i32> inreg %num, <2 x i32> inreg %d
;
; GFX1250-LABEL: s_mul_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s0, s0, s2
; GFX1250-NEXT: s_mul_i32 s1, s1, s3
@@ -722,6 +734,8 @@ define amdgpu_cs i33 @s_mul_i33(i33 inreg %num, i33 inreg %den) {
;
; GFX1250-LABEL: s_mul_i33:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_u64 s[0:1], s[0:1], s[2:3]
; GFX1250-NEXT: ; return to shader part epilog
@@ -789,6 +803,8 @@ define amdgpu_ps i64 @s_mul_i64(i64 inreg %num, i64 inreg %den) {
;
; GFX1250-LABEL: s_mul_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_u64 s[0:1], s[0:1], s[2:3]
; GFX1250-NEXT: ; return to shader part epilog
@@ -988,6 +1004,8 @@ define amdgpu_ps <3 x i32> @s_mul_i96(i96 inreg %num, i96 inreg %den) {
;
; GFX1250-LABEL: s_mul_i96:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s6, s0, s5
; GFX1250-NEXT: s_mul_i32 s7, s1, s4
@@ -1344,6 +1362,8 @@ define amdgpu_ps <4 x i32> @s_mul_i128(i128 inreg %num, i128 inreg %den) {
;
; GFX1250-LABEL: s_mul_i128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s9, s0, s6
; GFX1250-NEXT: s_mul_i32 s11, s1, s5
@@ -2640,6 +2660,8 @@ define amdgpu_ps <8 x i32> @s_mul_i256(i256 inreg %num, i256 inreg %den) {
;
; GFX1250-LABEL: s_mul_i256:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s17, s0, s10
; GFX1250-NEXT: s_mul_i32 s19, s1, s9
@@ -3704,6 +3726,8 @@ define amdgpu_ps void @s_mul_u64_zext_with_vregs(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: s_mul_u64_zext_with_vregs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v4, v[2:3], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -3817,6 +3841,8 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
;
; GFX1250-LABEL: s_mul_u64_zext_with_sregs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -3922,6 +3948,8 @@ define amdgpu_ps void @s_mul_u64_sext_with_vregs(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: s_mul_u64_sext_with_vregs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v4, v[2:3], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -4050,6 +4078,8 @@ define amdgpu_kernel void @s_mul_u64_sext_with_sregs(ptr addrspace(1) %out, ptr
;
; GFX1250-LABEL: s_mul_u64_sext_with_sregs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll
index 3e78f48e37a50..4118c5d34abe8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll
@@ -31,6 +31,8 @@ define amdgpu_ps i64 @s_mul_u64_u32(i32 inreg %a, i32 inreg %b) {
;
; GFX1250-LABEL: s_mul_u64_u32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: s_mov_b32 s1, 0
@@ -83,6 +85,8 @@ define amdgpu_ps i64 @s_mul_i64_i32(i32 inreg %a, i32 inreg %b) {
;
; GFX1250-LABEL: s_mul_i64_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s2, s1
; GFX1250-NEXT: s_ashr_i32 s1, s0, 31
diff --git a/llvm/test/CodeGen/AMDGPU/add-max.ll b/llvm/test/CodeGen/AMDGPU/add-max.ll
index 32ebcfe8e8751..754dce7575d7e 100644
--- a/llvm/test/CodeGen/AMDGPU/add-max.ll
+++ b/llvm/test/CodeGen/AMDGPU/add-max.ll
@@ -5,6 +5,8 @@
define amdgpu_ps float @add_max_u32_vvv(i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: add_max_u32_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_max_u32 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -17,6 +19,8 @@ define amdgpu_ps float @add_max_u32_vvv(i32 %a, i32 %b, i32 %c) {
define amdgpu_ps float @add_max_u32_svv(i32 inreg %a, i32 %b, i32 %c) {
; GCN-LABEL: add_max_u32_svv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_max_u32 v0, s0, v0, v1
; GCN-NEXT: ; return to shader part epilog
@@ -29,6 +33,8 @@ define amdgpu_ps float @add_max_u32_svv(i32 inreg %a, i32 %b, i32 %c) {
define amdgpu_ps float @add_max_u32_ssv(i32 inreg %a, i32 inreg %b, i32 %c) {
; GCN-LABEL: add_max_u32_ssv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_max_u32 v0, s0, s1, v0
; GCN-NEXT: ; return to shader part epilog
@@ -41,6 +47,8 @@ define amdgpu_ps float @add_max_u32_ssv(i32 inreg %a, i32 inreg %b, i32 %c) {
define amdgpu_ps float @add_max_u32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
; SDAG-LABEL: add_max_u32_sss:
; SDAG: ; %bb.0:
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: v_add_nc_u32_e64 v0, s0, s1 clamp
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -49,6 +57,8 @@ define amdgpu_ps float @add_max_u32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c
;
; GISEL-LABEL: add_max_u32_sss:
; GISEL: ; %bb.0:
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_add_nc_u32_e64 v0, s0, s1 clamp
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -65,6 +75,8 @@ define amdgpu_ps float @add_max_u32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c
define amdgpu_ps float @add_max_u32_vsi(i32 %a, i32 inreg %b) {
; GCN-LABEL: add_max_u32_vsi:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_max_u32 v0, v0, s0, 4
; GCN-NEXT: ; return to shader part epilog
@@ -77,6 +89,8 @@ define amdgpu_ps float @add_max_u32_vsi(i32 %a, i32 inreg %b) {
define amdgpu_ps float @add_max_u32_svl(i32 inreg %a, i32 %b) {
; GCN-LABEL: add_max_u32_svl:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_max_u32 v0, s0, v0, 0x64
; GCN-NEXT: ; return to shader part epilog
@@ -89,6 +103,8 @@ define amdgpu_ps float @add_max_u32_svl(i32 inreg %a, i32 %b) {
define amdgpu_ps float @add_max_u32_slv(i32 inreg %a, i32 %b, i32 %c) {
; GCN-LABEL: add_max_u32_slv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_max_u32 v0, s0, v0, v1
; GCN-NEXT: ; return to shader part epilog
@@ -101,6 +117,8 @@ define amdgpu_ps float @add_max_u32_slv(i32 inreg %a, i32 %b, i32 %c) {
define amdgpu_ps float @add_max_i32_vvv(i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: add_max_i32_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_max_i32 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -113,6 +131,8 @@ define amdgpu_ps float @add_max_i32_vvv(i32 %a, i32 %b, i32 %c) {
define amdgpu_ps float @add_min_u32_vvv(i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: add_min_u32_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_min_u32 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -125,6 +145,8 @@ define amdgpu_ps float @add_min_u32_vvv(i32 %a, i32 %b, i32 %c) {
define amdgpu_ps float @add_min_i32_vvv(i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: add_min_i32_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_add_min_i32 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -137,6 +159,8 @@ define amdgpu_ps float @add_min_i32_vvv(i32 %a, i32 %b, i32 %c) {
define amdgpu_ps float @add_max_v2u16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) {
; GCN-LABEL: add_max_v2u16_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_max_u16 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -149,6 +173,8 @@ define amdgpu_ps float @add_max_v2u16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16>
define amdgpu_ps float @add_max_v2u16_svv(<2 x i16> inreg %a, <2 x i16> %b, <2 x i16> %c) {
; GCN-LABEL: add_max_v2u16_svv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_max_u16 v0, s0, v0, v1
; GCN-NEXT: ; return to shader part epilog
@@ -161,6 +187,8 @@ define amdgpu_ps float @add_max_v2u16_svv(<2 x i16> inreg %a, <2 x i16> %b, <2 x
define amdgpu_ps float @add_max_v2u16_ssv(<2 x i16> inreg %a, <2 x i16> inreg %b, <2 x i16> %c) {
; GCN-LABEL: add_max_v2u16_ssv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_max_u16 v0, s0, s1, v0
; GCN-NEXT: ; return to shader part epilog
@@ -173,6 +201,8 @@ define amdgpu_ps float @add_max_v2u16_ssv(<2 x i16> inreg %a, <2 x i16> inreg %b
define amdgpu_ps float @add_max_v2u16_sss(<2 x i16> inreg %a, <2 x i16> inreg %b, <2 x i16> inreg %c) {
; SDAG-LABEL: add_max_v2u16_sss:
; SDAG: ; %bb.0:
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: v_pk_add_u16 v0, s0, s1 clamp
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -181,6 +211,8 @@ define amdgpu_ps float @add_max_v2u16_sss(<2 x i16> inreg %a, <2 x i16> inreg %b
;
; GISEL-LABEL: add_max_v2u16_sss:
; GISEL: ; %bb.0:
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_pk_add_u16 v0, s0, s1 clamp
; GISEL-NEXT: s_and_b32 s1, s2, 0xffff
@@ -204,6 +236,8 @@ define amdgpu_ps float @add_max_v2u16_sss(<2 x i16> inreg %a, <2 x i16> inreg %b
define amdgpu_ps float @add_max_v2u16_vsi(<2 x i16> %a, <2 x i16> inreg %b) {
; GCN-LABEL: add_max_v2u16_vsi:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_max_u16 v0, v0, s0, 4
; GCN-NEXT: ; return to shader part epilog
@@ -216,6 +250,8 @@ define amdgpu_ps float @add_max_v2u16_vsi(<2 x i16> %a, <2 x i16> inreg %b) {
define amdgpu_ps float @add_max_v2u16_svl(<2 x i16> inreg %a, <2 x i16> %b) {
; GCN-LABEL: add_max_v2u16_svl:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_max_u16 v0, s0, v0, 0x650064
; GCN-NEXT: ; return to shader part epilog
@@ -228,6 +264,8 @@ define amdgpu_ps float @add_max_v2u16_svl(<2 x i16> inreg %a, <2 x i16> %b) {
define amdgpu_ps float @add_max_v2u16_slv(<2 x i16> inreg %a, <2 x i16> %b) {
; GCN-LABEL: add_max_v2u16_slv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_max_u16 v0, 0x640064, s0, v0
; GCN-NEXT: ; return to shader part epilog
@@ -240,6 +278,8 @@ define amdgpu_ps float @add_max_v2u16_slv(<2 x i16> inreg %a, <2 x i16> %b) {
define amdgpu_ps float @add_max_v2s16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) {
; GCN-LABEL: add_max_v2s16_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_max_i16 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -252,6 +292,8 @@ define amdgpu_ps float @add_max_v2s16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16>
define amdgpu_ps float @add_min_v2u16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) {
; GCN-LABEL: add_min_v2u16_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_min_u16 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -264,6 +306,8 @@ define amdgpu_ps float @add_min_v2u16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16>
define amdgpu_ps float @add_min_v2s16_vvv(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c) {
; GCN-LABEL: add_min_v2s16_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_min_i16 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/add_u64.ll b/llvm/test/CodeGen/AMDGPU/add_u64.ll
index acab7d6e41950..72c56eb4c8fe3 100644
--- a/llvm/test/CodeGen/AMDGPU/add_u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_u64.ll
@@ -13,6 +13,8 @@ define amdgpu_ps <2 x float> @test_add_u64_vv(i64 %a, i64 %b) {
;
; GFX1250-LABEL: test_add_u64_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
; GFX1250-NEXT: ; return to shader part epilog
@@ -31,6 +33,8 @@ define amdgpu_ps <2 x float> @test_add_u64_vs(i64 %a, i64 inreg %b) {
;
; GFX1250-LABEL: test_add_u64_vs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -49,6 +53,8 @@ define amdgpu_ps <2 x float> @test_add_u64_sv(i64 inreg %a, i64 %b) {
;
; GFX1250-LABEL: test_add_u64_sv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -67,6 +73,8 @@ define amdgpu_ps <2 x float> @test_add_u64_ss(i64 inreg %a, i64 inreg %b) {
;
; GFX1250-LABEL: test_add_u64_ss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -87,6 +95,8 @@ define amdgpu_ps <2 x float> @test_add_u64_v_inline_lit(i64 %a) {
;
; GFX1250-LABEL: test_add_u64_v_inline_lit:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 5, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -105,6 +115,8 @@ define amdgpu_ps <2 x float> @test_add_u64_v_small_imm(i64 %a) {
;
; GFX1250-LABEL: test_add_u64_v_small_imm:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 0x1f4, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -123,6 +135,8 @@ define amdgpu_ps <2 x float> @test_add_u64_v_64bit_imm(i64 %a) {
;
; GFX1250-LABEL: test_add_u64_v_64bit_imm:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 0x13b9ac9ff, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -141,6 +155,8 @@ define amdgpu_ps <2 x float> @test_add_u64_s_small_imm(i64 inreg %a) {
;
; GFX1250-LABEL: test_add_u64_s_small_imm:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x1f4
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll
index 0635f50085555..29f01dffc0a31 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll
@@ -9,6 +9,8 @@ target triple = "amdgcn-amd-amdhsa"
define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %ptr) {
; GFX1250-SDAG-LABEL: use_private_to_flat_addrspacecast:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
@@ -27,6 +29,8 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %p
;
; GFX1250-GISEL-LABEL: use_private_to_flat_addrspacecast:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], src_flat_scratch_base_lo
@@ -52,6 +56,8 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %p
define amdgpu_kernel void @use_private_to_flat_addrspacecast_nonnull(ptr addrspace(5) %ptr) {
; GFX1250-SDAG-LABEL: use_private_to_flat_addrspacecast_nonnull:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
@@ -66,6 +72,8 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast_nonnull(ptr addrspa
;
; GFX1250-GISEL-LABEL: use_private_to_flat_addrspacecast_nonnull:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_mbcnt_lo_u32_b32 v2, -1, 0
@@ -88,6 +96,8 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast_nonnull(ptr addrspa
define amdgpu_kernel void @use_flat_to_private_addrspacecast(ptr %ptr) {
; GFX1250-LABEL: use_flat_to_private_addrspacecast:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -106,6 +116,8 @@ define amdgpu_kernel void @use_flat_to_private_addrspacecast(ptr %ptr) {
define amdgpu_kernel void @use_flat_to_private_addrspacecast_nonnull(ptr %ptr) {
; GFX1250-SDAG-LABEL: use_flat_to_private_addrspacecast_nonnull:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
@@ -117,6 +129,8 @@ define amdgpu_kernel void @use_flat_to_private_addrspacecast_nonnull(ptr %ptr) {
;
; GFX1250-GISEL-LABEL: use_flat_to_private_addrspacecast_nonnull:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 5c265a822d68e..dac7f642b419a 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -136,6 +136,8 @@ entry:
define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
; SDAG-LABEL: test_pipelined_loop:
; SDAG: ; %bb.0: ; %prolog
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -190,6 +192,8 @@ define amdgpu_kernel void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrsp
;
; GISEL-LABEL: test_pipelined_loop:
; GISEL: ; %bb.0: ; %prolog
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -307,6 +311,8 @@ epilog:
define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo, ptr addrspace(3) %lds, ptr addrspace(1) %bar, ptr addrspace(1) %out, i32 %n) {
; SDAG-LABEL: test_pipelined_loop_with_global:
; SDAG: ; %bb.0: ; %prolog
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: s_clause 0x1
; SDAG-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
@@ -393,6 +399,8 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
;
; GISEL-LABEL: test_pipelined_loop_with_global:
; GISEL: ; %bb.0: ; %prolog
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_clause 0x1
; GISEL-NEXT: s_load_b96 s[8:10], s[4:5], 0x24 nv
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 4b917977acd58..846fb981dcc39 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -14,6 +14,8 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_v(bfloat %v) {
;
; GFX1250-LABEL: v_test_cvt_bf16_f32_v:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-NEXT: ; return to shader part epilog
@@ -30,6 +32,8 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {
;
; GFX1250-LABEL: v_test_cvt_bf16_f32_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshl_b32 s0, s0, 16
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -66,6 +70,8 @@ define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_v(<2 x float> %src) {
;
; GFX1250-LABEL: v_test_cvt_v2f32_v2bf16_v:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
@@ -105,6 +111,8 @@ define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) {
;
; GFX1250-LABEL: v_test_cvt_v2f32_v2bf16_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
@@ -134,6 +142,8 @@ define amdgpu_ps float @v_test_cvt_f32_bf16_v(float %src) {
;
; GFX1250-LABEL: v_test_cvt_f32_bf16_v:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -211,6 +221,8 @@ define amdgpu_ps float @v_test_cvt_v2f64_v2bf16_v(<2 x double> %src) {
;
; GFX1250-LABEL: v_test_cvt_v2f64_v2bf16_v:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e32 v8, v[2:3]
; GFX1250-NEXT: v_cvt_f32_f64_e32 v9, v[0:1]
@@ -271,6 +283,8 @@ define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16(float %a, float %b) {
;
; GFX1250-LABEL: fptrunc_f32_f32_to_v2bf16:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
@@ -312,6 +326,8 @@ define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16_mods(float %a, float %b) {
;
; GFX1250-LABEL: fptrunc_f32_f32_to_v2bf16_mods:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, -v0, |v1|
; GFX1250-NEXT: ; return to shader part epilog
@@ -351,6 +367,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16(float %a, ptr %out) {
;
; GFX1250-LABEL: fptrunc_f32_to_bf16:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
@@ -388,6 +406,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_abs(float %a, ptr %out) {
;
; GFX1250-LABEL: fptrunc_f32_to_bf16_abs:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, |v0|, s0
@@ -426,6 +446,8 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_neg(float %a, ptr %out) {
;
; GFX1250-LABEL: fptrunc_f32_to_bf16_neg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, -v0, s0
@@ -479,6 +501,8 @@ define amdgpu_ps void @fptrunc_f64_to_bf16(double %a, ptr %out) {
;
; GFX1250-LABEL: fptrunc_f64_to_bf16:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -542,6 +566,8 @@ define amdgpu_ps void @fptrunc_f64_to_bf16_neg(double %a, ptr %out) {
;
; GFX1250-LABEL: fptrunc_f64_to_bf16_neg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e64 v6, -v[0:1]
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -607,6 +633,8 @@ define amdgpu_ps void @fptrunc_f64_to_bf16_abs(double %a, ptr %out) {
;
; GFX1250-LABEL: fptrunc_f64_to_bf16_abs:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-math.ll b/llvm/test/CodeGen/AMDGPU/bf16-math.ll
index fd1db62371aa9..0ef3936b26a0d 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-math.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-math.ll
@@ -6,6 +6,8 @@
define amdgpu_ps void @llvm_sqrt_bf16_v(ptr addrspace(1) %out, bfloat %src) {
; TRUE16-LABEL: llvm_sqrt_bf16_v:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_sqrt_bf16_e32 v2.l, v2.l
; TRUE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -13,6 +15,8 @@ define amdgpu_ps void @llvm_sqrt_bf16_v(ptr addrspace(1) %out, bfloat %src) {
;
; FAKE16-LABEL: llvm_sqrt_bf16_v:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_sqrt_bf16_e32 v2, v2
; FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -25,6 +29,8 @@ define amdgpu_ps void @llvm_sqrt_bf16_v(ptr addrspace(1) %out, bfloat %src) {
define amdgpu_ps void @llvm_sqrt_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {
; TRUE16-LABEL: llvm_sqrt_bf16_s:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_sqrt_bf16_e32 v2.l, s0
; TRUE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -32,6 +38,8 @@ define amdgpu_ps void @llvm_sqrt_bf16_s(ptr addrspace(1) %out, bfloat inreg %src
;
; FAKE16-LABEL: llvm_sqrt_bf16_s:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_sqrt_bf16_e32 v2, s0
; FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -44,6 +52,8 @@ define amdgpu_ps void @llvm_sqrt_bf16_s(ptr addrspace(1) %out, bfloat inreg %src
define amdgpu_ps void @v_test_add_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_add_v2bf16_vv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, v2, v3
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -56,6 +66,8 @@ define amdgpu_ps void @v_test_add_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_add_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_add_v2bf16_vs:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, v2, s0
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -68,6 +80,8 @@ define amdgpu_ps void @v_test_add_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_add_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_add_v2bf16_ss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, s0, s1
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -80,6 +94,8 @@ define amdgpu_ps void @v_test_add_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_add_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_add_v2bf16_vc:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, v2, 2.0 op_sel_hi:[1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -92,6 +108,8 @@ define amdgpu_ps void @v_test_add_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_add_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_add_v2bf16_vl:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, 0x42c83f80, v2
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -104,6 +122,8 @@ define amdgpu_ps void @v_test_add_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_sub_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_sub_v2bf16_vv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, v2, v3 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -116,6 +136,8 @@ define amdgpu_ps void @v_test_sub_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_sub_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_sub_v2bf16_vs:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, v2, s0 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -128,6 +150,8 @@ define amdgpu_ps void @v_test_sub_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_sub_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_sub_v2bf16_ss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -140,6 +164,8 @@ define amdgpu_ps void @v_test_sub_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_sub_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_vc:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, v2, -2.0 op_sel_hi:[1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -152,6 +178,8 @@ define amdgpu_ps void @v_test_sub_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_sub_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_vl:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, 0xc2c8bf80, v2
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -164,6 +192,8 @@ define amdgpu_ps void @v_test_sub_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_sub_v2bf16_lv(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_lv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, 0x42c83f80, v2 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -176,6 +206,8 @@ define amdgpu_ps void @v_test_sub_v2bf16_lv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_sub_v2bf16_iv(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_iv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_add_bf16 v2, v2, 1.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -188,6 +220,8 @@ define amdgpu_ps void @v_test_sub_v2bf16_iv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_mul_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_mul_v2bf16_vv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_mul_bf16 v2, v2, v3
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -200,6 +234,8 @@ define amdgpu_ps void @v_test_mul_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_mul_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_mul_v2bf16_vs:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_mul_bf16 v2, v2, s0
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -212,6 +248,8 @@ define amdgpu_ps void @v_test_mul_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_mul_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_mul_v2bf16_ss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_mul_bf16 v2, s0, s1
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -226,6 +264,8 @@ define amdgpu_ps void @v_test_mul_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_mul_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_mul_v2bf16_vc:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_mul_bf16 v2, v2, 0.5 op_sel_hi:[1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -238,6 +278,8 @@ define amdgpu_ps void @v_test_mul_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_mul_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_mul_v2bf16_vl:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_mul_bf16 v2, 0x42c83f80, v2
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -250,6 +292,8 @@ define amdgpu_ps void @v_test_mul_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_min_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_min_v2bf16_vv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_min_num_bf16 v2, v2, v3
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -262,6 +306,8 @@ define amdgpu_ps void @v_test_min_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_min_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_min_v2bf16_vs:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_min_num_bf16 v2, v2, s0
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -274,6 +320,8 @@ define amdgpu_ps void @v_test_min_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_min_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_min_v2bf16_ss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_min_num_bf16 v2, s0, s1
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -286,6 +334,8 @@ define amdgpu_ps void @v_test_min_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_min_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_min_v2bf16_vc:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_min_num_bf16 v2, v2, 0.5 op_sel_hi:[1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -298,6 +348,8 @@ define amdgpu_ps void @v_test_min_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_min_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_min_v2bf16_vl:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_min_num_bf16 v2, 0x42c83f80, v2
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -310,6 +362,8 @@ define amdgpu_ps void @v_test_min_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_max_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_max_v2bf16_vv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_max_num_bf16 v2, v2, v3
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -322,6 +376,8 @@ define amdgpu_ps void @v_test_max_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_max_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_max_v2bf16_vs:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_max_num_bf16 v2, v2, s0
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -334,6 +390,8 @@ define amdgpu_ps void @v_test_max_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_max_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_max_v2bf16_ss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_max_num_bf16 v2, s0, s1
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -346,6 +404,8 @@ define amdgpu_ps void @v_test_max_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_max_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_max_v2bf16_vc:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_max_num_bf16 v2, v2, 0.5 op_sel_hi:[1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -358,6 +418,8 @@ define amdgpu_ps void @v_test_max_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_max_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_max_v2bf16_vl:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_max_num_bf16 v2, 0x42c83f80, v2
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -370,12 +432,16 @@ define amdgpu_ps void @v_test_max_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps bfloat @test_clamp_bf16(bfloat %src) {
; TRUE16-LABEL: test_clamp_bf16:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_pk_max_num_bf16 v0, v0, v0 op_sel_hi:[0,0] clamp
; TRUE16-NEXT: ; return to shader part epilog
;
; FAKE16-LABEL: test_clamp_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp
; FAKE16-NEXT: ; return to shader part epilog
@@ -387,6 +453,8 @@ define amdgpu_ps bfloat @test_clamp_bf16(bfloat %src) {
define amdgpu_ps bfloat @test_clamp_bf16_s(bfloat inreg %src) {
; TRUE16-LABEL: test_clamp_bf16_s:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -395,6 +463,8 @@ define amdgpu_ps bfloat @test_clamp_bf16_s(bfloat inreg %src) {
;
; FAKE16-LABEL: test_clamp_bf16_s:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_pk_max_num_bf16 v0, s0, s0 clamp
; FAKE16-NEXT: ; return to shader part epilog
@@ -406,6 +476,8 @@ define amdgpu_ps bfloat @test_clamp_bf16_s(bfloat inreg %src) {
define amdgpu_ps float @test_clamp_bf16_hi16(<2 x bfloat> %src) {
; TRUE16-LABEL: test_clamp_bf16_hi16:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -415,6 +487,8 @@ define amdgpu_ps float @test_clamp_bf16_hi16(<2 x bfloat> %src) {
;
; FAKE16-LABEL: test_clamp_bf16_hi16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -431,6 +505,8 @@ define amdgpu_ps float @test_clamp_bf16_hi16(<2 x bfloat> %src) {
define amdgpu_ps float @test_clamp_bf16_hi16_s(<2 x bfloat> inreg %src) {
; TRUE16-LABEL: test_clamp_bf16_hi16_s:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: s_lshr_b32 s0, s0, 16
; TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -442,6 +518,8 @@ define amdgpu_ps float @test_clamp_bf16_hi16_s(<2 x bfloat> inreg %src) {
;
; FAKE16-LABEL: test_clamp_bf16_hi16_s:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_lshr_b32 s0, s0, 16
; FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -458,6 +536,8 @@ define amdgpu_ps float @test_clamp_bf16_hi16_s(<2 x bfloat> inreg %src) {
define amdgpu_ps float @test_clamp_v2bf16(<2 x bfloat> %src) {
; GCN-LABEL: test_clamp_v2bf16:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp
; GCN-NEXT: ; return to shader part epilog
@@ -470,6 +550,8 @@ define amdgpu_ps float @test_clamp_v2bf16(<2 x bfloat> %src) {
define amdgpu_ps float @test_clamp_v2bf16_s(<2 x bfloat> inreg %src) {
; GCN-LABEL: test_clamp_v2bf16_s:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_max_num_bf16 v0, s0, s0 clamp
; GCN-NEXT: ; return to shader part epilog
@@ -482,6 +564,8 @@ define amdgpu_ps float @test_clamp_v2bf16_s(<2 x bfloat> inreg %src) {
define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) {
; TRUE16-LABEL: test_clamp_bf16_folding:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_exp_bf16_e32 v0.l, v0.l
; TRUE16-NEXT: v_nop
@@ -491,6 +575,8 @@ define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) {
;
; FAKE16-LABEL: test_clamp_bf16_folding:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_exp_bf16_e64 v0, v0 clamp
; FAKE16-NEXT: ; return to shader part epilog
@@ -503,6 +589,8 @@ define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) {
define amdgpu_ps float @test_clamp_v2bf16_folding(<2 x bfloat> %src0, <2 x bfloat> %src1) {
; GCN-LABEL: test_clamp_v2bf16_folding:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_mul_bf16 v0, v0, v1 clamp
; GCN-NEXT: ; return to shader part epilog
@@ -516,6 +604,8 @@ define amdgpu_ps float @test_clamp_v2bf16_folding(<2 x bfloat> %src0, <2 x bfloa
define amdgpu_ps void @v_test_mul_add_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
; GCN-LABEL: v_test_mul_add_v2bf16_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_fma_bf16 v2, v2, v3, v4
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -529,6 +619,8 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfl
define amdgpu_ps void @v_test_mul_add_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_mul_add_v2bf16_vss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, s1
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -542,6 +634,8 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_vss(ptr addrspace(1) %out, <2 x bfl
define amdgpu_ps void @v_test_mul_add_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_mul_add_v2bf16_sss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v2, s2
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -557,6 +651,8 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_sss(ptr addrspace(1) %out, <2 x bfl
define amdgpu_ps void @v_test_mul_add_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_mul_add_v2bf16_vsc:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, 0.5 op_sel_hi:[1,1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -570,6 +666,8 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfl
define amdgpu_ps void @v_test_mul_add_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_mul_add_v2bf16_vll:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_mov_b32 s0, 0x43484000
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -585,6 +683,8 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_vll(ptr addrspace(1) %out, <2 x bfl
define amdgpu_ps void @v_test_fma_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
; GCN-LABEL: v_test_fma_v2bf16_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_fma_bf16 v2, v2, v3, v4
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -597,6 +697,8 @@ define amdgpu_ps void @v_test_fma_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_fma_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_fma_v2bf16_vss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, s1
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -609,6 +711,8 @@ define amdgpu_ps void @v_test_fma_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_fma_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_fma_v2bf16_sss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v2, s2
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -623,6 +727,8 @@ define amdgpu_ps void @v_test_fma_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_fma_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_fma_v2bf16_vsc:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, 0.5 op_sel_hi:[1,1,0]
; GCN-NEXT: global_store_b32 v[0:1], v2, off
@@ -635,6 +741,8 @@ define amdgpu_ps void @v_test_fma_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @v_test_fma_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_fma_v2bf16_vll:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_mov_b32 s0, 0x42c83f80
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -649,6 +757,8 @@ define amdgpu_ps void @v_test_fma_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat>
define amdgpu_ps void @llvm_log2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
; TRUE16-LABEL: llvm_log2_bf16_v:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_log_bf16_e32 v2.l, v2.l
; TRUE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -656,6 +766,8 @@ define amdgpu_ps void @llvm_log2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
;
; FAKE16-LABEL: llvm_log2_bf16_v:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_log_bf16_e32 v2, v2
; FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -668,6 +780,8 @@ define amdgpu_ps void @llvm_log2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
define amdgpu_ps void @llvm_log2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {
; TRUE16-LABEL: llvm_log2_bf16_s:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_log_bf16_e32 v2.l, s0
; TRUE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -675,6 +789,8 @@ define amdgpu_ps void @llvm_log2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src
;
; FAKE16-LABEL: llvm_log2_bf16_s:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_log_bf16_e32 v2, s0
; FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -687,6 +803,8 @@ define amdgpu_ps void @llvm_log2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src
define amdgpu_ps void @llvm_exp2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
; TRUE16-LABEL: llvm_exp2_bf16_v:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_exp_bf16_e32 v2.l, v2.l
; TRUE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -694,6 +812,8 @@ define amdgpu_ps void @llvm_exp2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
;
; FAKE16-LABEL: llvm_exp2_bf16_v:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_exp_bf16_e32 v2, v2
; FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -706,6 +826,8 @@ define amdgpu_ps void @llvm_exp2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
define amdgpu_ps void @llvm_exp2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {
; TRUE16-LABEL: llvm_exp2_bf16_s:
; TRUE16: ; %bb.0:
+; TRUE16-NEXT: global_wb
+; TRUE16-NEXT: v_nop
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_exp_bf16_e32 v2.l, s0
; TRUE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -713,6 +835,8 @@ define amdgpu_ps void @llvm_exp2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src
;
; FAKE16-LABEL: llvm_exp2_bf16_s:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: v_exp_bf16_e32 v2, s0
; FAKE16-NEXT: global_store_b16 v[0:1], v2, off
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index dc125ec5840ae..fd2cfb5b59dd0 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -18803,6 +18803,8 @@ define amdgpu_ps i32 @s_fabs_bf16(bfloat inreg %a) #0 {
;
; GFX1250-LABEL: s_fabs_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -18918,6 +18920,8 @@ define amdgpu_ps i32 @s_fneg_bf16(bfloat inreg %a) #0 {
;
; GFX1250-LABEL: s_fneg_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_xor_b32 s0, s0, 0x8000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -19032,6 +19036,8 @@ define amdgpu_ps i32 @s_fneg_fabs_bf16(bfloat inreg %a) #0 {
;
; GFX1250-LABEL: s_fneg_fabs_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_bitset1_b32 s0, 15
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -41347,6 +41353,8 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) #0
;
; GFX1250TRUE16-LABEL: s_select_bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: global_wb
+; GFX1250TRUE16-NEXT: v_nop
; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
@@ -41359,6 +41367,8 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) #0
;
; GFX1250FAKE16-LABEL: s_select_bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: global_wb
+; GFX1250FAKE16-NEXT: v_nop
; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -41504,6 +41514,8 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250TRUE16-LABEL: s_select_v2bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: global_wb
+; GFX1250TRUE16-NEXT: v_nop
; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -41519,6 +41531,8 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250FAKE16-LABEL: s_select_v2bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: global_wb
+; GFX1250FAKE16-NEXT: v_nop
; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -41666,6 +41680,8 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250TRUE16-LABEL: s_vselect_v2bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: global_wb
+; GFX1250TRUE16-NEXT: v_nop
; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: s_lshr_b32 s3, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -41682,6 +41698,8 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250FAKE16-LABEL: s_vselect_v2bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: global_wb
+; GFX1250FAKE16-NEXT: v_nop
; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
@@ -42468,6 +42486,8 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat>
;
; GFX1250-LABEL: s_select_v3bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -42585,6 +42605,8 @@ define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250-LABEL: s_select_v4bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s0
; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -42818,6 +42840,8 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250TRUE16-LABEL: s_vselect_v4bf16:
; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: global_wb
+; GFX1250TRUE16-NEXT: v_nop
; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: s_lshr_b32 s7, s1, 16
; GFX1250TRUE16-NEXT: s_lshr_b32 s9, s0, 16
@@ -42842,6 +42866,8 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250FAKE16-LABEL: s_vselect_v4bf16:
; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: global_wb
+; GFX1250FAKE16-NEXT: v_nop
; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: s_lshr_b32 s4, s1, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/bitop3.ll b/llvm/test/CodeGen/AMDGPU/bitop3.ll
index a4c5d5649c226..d052af4560eb0 100644
--- a/llvm/test/CodeGen/AMDGPU/bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitop3.ll
@@ -16,6 +16,8 @@ define amdgpu_ps float @not_and_not_and_not_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: not_and_not_and_not_and:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:1
; GFX1250-NEXT: ; return to shader part epilog
@@ -36,6 +38,8 @@ define amdgpu_ps float @not_and_not_and_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: not_and_not_and_and:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:2
; GFX1250-NEXT: ; return to shader part epilog
@@ -55,6 +59,8 @@ define amdgpu_ps float @not_and_and_not_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: not_and_and_not_and:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:4
; GFX1250-NEXT: ; return to shader part epilog
@@ -80,12 +86,16 @@ define amdgpu_ps float @not_and_and_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-SDAG-LABEL: not_and_and_and:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:8
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: not_and_and_and:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_bitop3_b32 v0, v0, v2, v0 bitop3:0xc
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -106,6 +116,8 @@ define amdgpu_ps float @and_not_and_not_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: and_not_and_not_and:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:0x10
; GFX1250-NEXT: ; return to shader part epilog
@@ -131,12 +143,16 @@ define amdgpu_ps float @and_not_and_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-SDAG-LABEL: and_not_and_and:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:0x20
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: and_not_and_and:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -163,12 +179,16 @@ define amdgpu_ps float @and_and_not_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-SDAG-LABEL: and_and_not_and:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:0x40
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: and_and_not_and:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_bitop3_b32 v0, v0, v2, v0 bitop3:0x30
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -189,6 +209,8 @@ define amdgpu_ps float @and_and_and(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: and_and_and:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:0x80
; GFX1250-NEXT: ; return to shader part epilog
@@ -208,6 +230,8 @@ define amdgpu_ps float @test_12(i32 %a, i32 %b) {
;
; GFX1250-LABEL: test_12:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v0 bitop3:0xc
; GFX1250-NEXT: ; return to shader part epilog
@@ -231,12 +255,16 @@ define amdgpu_ps float @test_63(i32 %a, i32 %b) {
;
; GFX1250-SDAG-LABEL: test_63:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_bitop3_b32 v0, v0, v1, v0 bitop3:0x3f
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: test_63:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_not_b32_e32 v1, v1
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -257,6 +285,8 @@ define amdgpu_ps float @test_59(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: test_59:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:0x3b
; GFX1250-NEXT: ; return to shader part epilog
@@ -283,12 +313,16 @@ define amdgpu_ps float @test_126(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-SDAG-LABEL: test_126:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_bitop3_b32 v0, v0, v2, v1 bitop3:0x7e
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: test_126:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, v0, v1
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v2
@@ -323,12 +357,16 @@ define amdgpu_ps float @test_12_src_overflow(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-SDAG-LABEL: test_12_src_overflow:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_bitop3_b32 v0, v0, v1, v0 bitop3:0xc
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: test_12_src_overflow:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_not_b32_e32 v3, v0
; GFX1250-GISEL-NEXT: v_bitop3_b32 v0, v0, v2, v0 bitop3:0xc
@@ -371,6 +409,8 @@ define amdgpu_ps float @test_100_src_overflow(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-SDAG-LABEL: test_100_src_overflow:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_bitop3_b32 v3, v1, v2, v0 bitop3:0x10
; GFX1250-SDAG-NEXT: v_bitop3_b32 v4, v0, v2, v1 bitop3:0x40
@@ -381,6 +421,8 @@ define amdgpu_ps float @test_100_src_overflow(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-GISEL-LABEL: test_100_src_overflow:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_or_b32_e32 v3, v2, v0
; GFX1250-GISEL-NEXT: v_bitop3_b32 v4, v0, v1, v0 bitop3:0x30
@@ -418,6 +460,8 @@ define amdgpu_ps float @test_xor3(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: test_xor3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_xor3_b32 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
@@ -435,6 +479,8 @@ define amdgpu_ps float @test_or3(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: test_or3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
@@ -452,6 +498,8 @@ define amdgpu_ps float @test_and_or(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: test_and_or:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_and_or_b32 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
@@ -473,6 +521,8 @@ define amdgpu_ps float @uniform_3_op(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
;
; GFX1250-LABEL: uniform_3_op:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_not1_b32 s0, s2, s0
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -503,6 +553,8 @@ define amdgpu_ps float @uniform_4_op(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
;
; GFX1250-SDAG-LABEL: uniform_4_op:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -511,6 +563,8 @@ define amdgpu_ps float @uniform_4_op(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
;
; GFX1250-GISEL-LABEL: uniform_4_op:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_and_not1_b32 s0, s2, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -535,24 +589,32 @@ define amdgpu_ps half @not_and_not_and_not_and_b16(i16 %a, i16 %b, i16 %c) {
;
; GFX1250-SDAG-FAKE16-LABEL: not_and_not_and_not_and_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:1
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: not_and_not_and_not_and_b16:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, v2.l bitop3:1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: not_and_not_and_not_and_b16:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:1
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: not_and_not_and_not_and_b16:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, v2.l bitop3:1
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -573,24 +635,32 @@ define amdgpu_ps half @not_and_not_and_and_b16(i16 %a, i16 %b, i16 %c) {
;
; GFX1250-SDAG-FAKE16-LABEL: not_and_not_and_and_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: not_and_not_and_and_b16:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, v2.l bitop3:2
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: not_and_not_and_and_b16:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:2
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: not_and_not_and_and_b16:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, v2.l bitop3:2
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -610,24 +680,32 @@ define amdgpu_ps half @not_and_and_not_and_b16(i16 %a, i16 %b, i16 %c) {
;
; GFX1250-SDAG-FAKE16-LABEL: not_and_and_not_and_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:4
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: not_and_and_not_and_b16:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, v2.l bitop3:4
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: not_and_and_not_and_b16:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:4
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: not_and_and_not_and_b16:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, v2.l bitop3:4
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -653,18 +731,24 @@ define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_xor3_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0x96
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: test_xor3_b16:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0x96
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: test_xor3_b16:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_xor3_b32 v0, v0, v1, v2
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -687,18 +771,24 @@ define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_or3_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xfe
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: test_or3_b16:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xfe
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: test_or3_b16:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -721,18 +811,24 @@ define amdgpu_ps half @test_and_or_b16(i16 %a, i16 %b, i16 %c) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_and_or_b16:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_bitop3_b16 v0, v0, v2, v1 bitop3:0xec
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: test_and_or_b16:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xec
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: test_and_or_b16:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
; GFX1250-GISEL-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
index e1fbd4e88be46..c37509c27940c 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
@@ -23,6 +23,8 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addrspace(1) %arg, i32 %cnd) #0 {
; GCN-LABEL: uniform_conditional_max_short_forward_branch:
; GCN: ; %bb.0: ; %bb
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -50,6 +52,8 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
;
; GCN-ADD-PC64-LABEL: uniform_conditional_max_short_forward_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
@@ -114,6 +118,8 @@ bb3:
define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrspace(1) %arg, i32 %cnd) #0 {
; GCN-LABEL: uniform_conditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -142,6 +148,8 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
;
; GCN-ADD-PC64-LABEL: uniform_conditional_min_long_forward_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb0
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
@@ -209,6 +217,8 @@ bb3:
define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr addrspace(1) %arg, float %cnd) #0 {
; GCN-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GCN: ; %bb.0: ; %bb0
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -238,6 +248,8 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
;
; GCN-ADD-PC64-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb0
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
@@ -306,6 +318,8 @@ bb3:
define amdgpu_kernel void @min_long_forward_vbranch(ptr addrspace(1) %arg) #0 {
; GCN-LABEL: min_long_forward_vbranch:
; GCN: ; %bb.0: ; %bb
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -341,6 +355,8 @@ define amdgpu_kernel void @min_long_forward_vbranch(ptr addrspace(1) %arg) #0 {
;
; GCN-ADD-PC64-LABEL: min_long_forward_vbranch:
; GCN-ADD-PC64: ; %bb.0: ; %bb
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GCN-ADD-PC64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -425,6 +441,8 @@ bb3:
define amdgpu_kernel void @long_backward_sbranch(ptr addrspace(1) %arg) #0 {
; GCN-LABEL: long_backward_sbranch:
; GCN: ; %bb.0: ; %bb
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_mov_b32 s0, 0
; GCN-NEXT: .LBB4_1: ; %bb2
@@ -449,6 +467,8 @@ define amdgpu_kernel void @long_backward_sbranch(ptr addrspace(1) %arg) #0 {
;
; GCN-ADD-PC64-LABEL: long_backward_sbranch:
; GCN-ADD-PC64: ; %bb.0: ; %bb
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_mov_b32 s0, 0
; GCN-ADD-PC64-NEXT: .LBB4_1: ; %bb2
@@ -510,6 +530,8 @@ bb3:
define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr addrspace(1) %arg, i32 %arg1) {
; GCN-LABEL: uniform_unconditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -550,6 +572,8 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
;
; GCN-ADD-PC64-LABEL: uniform_unconditional_min_long_forward_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb0
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
@@ -648,6 +672,8 @@ bb4:
define amdgpu_kernel void @uniform_unconditional_min_long_backward_branch(ptr addrspace(1) %arg, i32 %arg1) {
; GCN-LABEL: uniform_unconditional_min_long_backward_branch:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_mov_b32 vcc_lo, exec_lo
; GCN-NEXT: .LBB6_1: ; %loop
@@ -671,6 +697,8 @@ define amdgpu_kernel void @uniform_unconditional_min_long_backward_branch(ptr ad
;
; GCN-ADD-PC64-LABEL: uniform_unconditional_min_long_backward_branch:
; GCN-ADD-PC64: ; %bb.0: ; %entry
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_mov_b32 vcc_lo, exec_lo
; GCN-ADD-PC64-NEXT: .LBB6_1: ; %loop
@@ -726,6 +754,8 @@ loop:
define amdgpu_kernel void @expand_requires_expand(i32 %cond0) #0 {
; GCN-LABEL: expand_requires_expand:
; GCN: ; %bb.0: ; %bb0
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -769,6 +799,8 @@ define amdgpu_kernel void @expand_requires_expand(i32 %cond0) #0 {
;
; GCN-ADD-PC64-LABEL: expand_requires_expand:
; GCN-ADD-PC64: ; %bb.0: ; %bb0
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
@@ -876,6 +908,8 @@ bb3:
define amdgpu_kernel void @uniform_inside_divergent(ptr addrspace(1) %out, i32 %cond) #0 {
; GCN-LABEL: uniform_inside_divergent:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: s_mov_b32 s3, exec_lo
@@ -906,6 +940,8 @@ define amdgpu_kernel void @uniform_inside_divergent(ptr addrspace(1) %out, i32 %
;
; GCN-ADD-PC64-LABEL: uniform_inside_divergent:
; GCN-ADD-PC64: ; %bb.0: ; %entry
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-ADD-PC64-NEXT: s_mov_b32 s3, exec_lo
@@ -982,6 +1018,8 @@ endif:
define amdgpu_kernel void @analyze_mask_branch() #0 {
; GCN-LABEL: analyze_mask_branch:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_mov_b32 s0, exec_lo
; GCN-NEXT: ;;#ASMSTART
@@ -1031,6 +1069,8 @@ define amdgpu_kernel void @analyze_mask_branch() #0 {
;
; GCN-ADD-PC64-LABEL: analyze_mask_branch:
; GCN-ADD-PC64: ; %bb.0: ; %entry
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_mov_b32 s0, exec_lo
; GCN-ADD-PC64-NEXT: ;;#ASMSTART
@@ -1142,6 +1182,8 @@ ret:
define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i64 %arg5) #0 {
; GCN-LABEL: long_branch_hang:
; GCN: ; %bb.0: ; %bb
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
; GCN-NEXT: s_mov_b32 s8, -1
@@ -1219,6 +1261,8 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
;
; GCN-ADD-PC64-LABEL: long_branch_hang:
; GCN-ADD-PC64: ; %bb.0: ; %bb
+; GCN-ADD-PC64-NEXT: global_wb
+; GCN-ADD-PC64-NEXT: v_nop
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-ADD-PC64-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
; GCN-ADD-PC64-NEXT: s_mov_b32 s8, -1
diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
index 411468e07e131..722b6170b7b8e 100644
--- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -39,6 +39,8 @@ define spir_kernel void @kernel(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: kernel:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -84,6 +86,8 @@ define amdgpu_ps half @ps_ret_cc_f16(half %arg0) {
;
; GFX1250-TRUE16-LABEL: ps_ret_cc_f16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
@@ -91,6 +95,8 @@ define amdgpu_ps half @ps_ret_cc_f16(half %arg0) {
;
; GFX1250-FAKE16-LABEL: ps_ret_cc_f16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_add_f16_e32 v0, 1.0, v0
@@ -124,6 +130,8 @@ define amdgpu_ps half @ps_ret_cc_inreg_f16(half inreg %arg0) {
;
; GFX1250-LABEL: ps_ret_cc_inreg_f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_f16 s0, s0, 1.0
@@ -284,6 +292,8 @@ define amdgpu_kernel void @call_coldcc() #0 {
;
; GFX1250-LABEL: call_coldcc:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s32, 0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b64 s[10:11], s[6:7]
@@ -396,6 +406,8 @@ define amdgpu_kernel void @call_fastcc() #0 {
;
; GFX1250-LABEL: call_fastcc:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s32, 0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b64 s[10:11], s[6:7]
@@ -442,6 +454,8 @@ define amdgpu_cs half @cs_mesa(half %arg0) {
;
; GFX1250-TRUE16-LABEL: cs_mesa:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
@@ -449,6 +463,8 @@ define amdgpu_cs half @cs_mesa(half %arg0) {
;
; GFX1250-FAKE16-LABEL: cs_mesa:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_add_f16_e32 v0, 1.0, v0
@@ -483,6 +499,8 @@ define amdgpu_ps half @ps_mesa_f16(half %arg0) {
;
; GFX1250-TRUE16-LABEL: ps_mesa_f16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
@@ -490,6 +508,8 @@ define amdgpu_ps half @ps_mesa_f16(half %arg0) {
;
; GFX1250-FAKE16-LABEL: ps_mesa_f16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_add_f16_e32 v0, 1.0, v0
@@ -524,6 +544,8 @@ define amdgpu_vs half @vs_mesa(half %arg0) {
;
; GFX1250-TRUE16-LABEL: vs_mesa:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
@@ -531,6 +553,8 @@ define amdgpu_vs half @vs_mesa(half %arg0) {
;
; GFX1250-FAKE16-LABEL: vs_mesa:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_add_f16_e32 v0, 1.0, v0
@@ -565,6 +589,8 @@ define amdgpu_gs half @gs_mesa(half %arg0) {
;
; GFX1250-TRUE16-LABEL: gs_mesa:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
@@ -572,6 +598,8 @@ define amdgpu_gs half @gs_mesa(half %arg0) {
;
; GFX1250-FAKE16-LABEL: gs_mesa:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_add_f16_e32 v0, 1.0, v0
@@ -606,6 +634,8 @@ define amdgpu_hs half @hs_mesa(half %arg0) {
;
; GFX1250-TRUE16-LABEL: hs_mesa:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
@@ -613,6 +643,8 @@ define amdgpu_hs half @hs_mesa(half %arg0) {
;
; GFX1250-FAKE16-LABEL: hs_mesa:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_add_f16_e32 v0, 1.0, v0
@@ -652,6 +684,8 @@ define amdgpu_ps <2 x half> @ps_mesa_v2f16(<2 x half> %arg0) {
;
; GFX1250-LABEL: ps_mesa_v2f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
@@ -691,6 +725,8 @@ define amdgpu_ps <2 x half> @ps_mesa_inreg_v2f16(<2 x half> inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_v2f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_add_f16 v0, s0, 1.0 op_sel_hi:[1,0]
@@ -728,6 +764,8 @@ define amdgpu_ps void @ps_mesa_v2i16(<2 x i16> %arg0) {
;
; GFX1250-LABEL: ps_mesa_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
; GFX1250-NEXT: global_store_b32 v[0:1], v0, off
@@ -770,6 +808,8 @@ define amdgpu_ps void @ps_mesa_inreg_v2i16(<2 x i16> inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_add_u16 v0, s0, 1 op_sel_hi:[1,0]
; GFX1250-NEXT: global_store_b32 v[0:1], v0, off
@@ -823,6 +863,8 @@ define amdgpu_ps <4 x half> @ps_mesa_v4f16(<4 x half> %arg0) {
;
; GFX1250-LABEL: ps_mesa_v4f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_add_f16 v0, v0, 1.0 op_sel_hi:[1,0]
@@ -878,6 +920,8 @@ define amdgpu_ps <4 x half> @ps_mesa_inreg_v4f16(<4 x half> inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_v4f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_pk_add_f16 v0, s0, 1.0 op_sel_hi:[1,0]
@@ -927,6 +971,8 @@ define amdgpu_ps void @ps_mesa_inreg_v3i32(<3 x i32> inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_v3i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_co_i32 s2, s2, 3
; GFX1250-NEXT: s_add_co_i32 s0, s0, 1
@@ -971,6 +1017,8 @@ define amdgpu_ps void @ps_mesa_inreg_v3f32(<3 x float> inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_v3f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_f32 s0, s0, 1.0
; GFX1250-NEXT: s_add_f32 s1, s1, 2.0
@@ -1038,6 +1086,8 @@ define amdgpu_ps void @ps_mesa_inreg_v5i32(<5 x i32> inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_v5i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_co_i32 s1, s1, 2
; GFX1250-NEXT: s_add_co_i32 s4, s4, 5
@@ -1095,6 +1145,8 @@ define amdgpu_ps void @ps_mesa_inreg_v5f32(<5 x float> inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_v5f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_f32 s4, s4, 0.5
; GFX1250-NEXT: s_add_f32 s0, s0, 1.0
@@ -1144,6 +1196,8 @@ define amdgpu_ps void @ps_mesa_v3i32(<3 x i32> %arg0) {
;
; GFX1250-LABEL: ps_mesa_v3i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_add_nc_u32 v2, 3, v2 :: v_dual_add_nc_u32 v1, 2, v1
; GFX1250-NEXT: v_add_nc_u32_e32 v0, 1, v0
@@ -1183,6 +1237,8 @@ define amdgpu_ps void @ps_mesa_v3f32(<3 x float> %arg0) {
;
; GFX1250-LABEL: ps_mesa_v3f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_add_f32 v2, 4.0, v2 :: v_dual_add_f32 v1, 2.0, v1
; GFX1250-NEXT: v_add_f32_e32 v0, 1.0, v0
@@ -1232,6 +1288,8 @@ define amdgpu_ps void @ps_mesa_v5i32(<5 x i32> %arg0) {
;
; GFX1250-LABEL: ps_mesa_v5i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_add_nc_u32 v1, 2, v1 :: v_dual_add_nc_u32 v4, 5, v4
; GFX1250-NEXT: v_dual_add_nc_u32 v0, 1, v0 :: v_dual_add_nc_u32 v3, 4, v3
@@ -1282,6 +1340,8 @@ define amdgpu_ps void @ps_mesa_v5f32(<5 x float> %arg0) {
;
; GFX1250-LABEL: ps_mesa_v5f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_add_f32 v1, 2.0, v1 :: v_dual_add_f32 v4, 0.5, v4
; GFX1250-NEXT: v_dual_add_f32 v0, 1.0, v0 :: v_dual_add_f32 v3, -1.0, v3
@@ -1324,6 +1384,8 @@ define amdgpu_ps void @ps_mesa_i16(i16 %arg0) {
;
; GFX1250-TRUE16-LABEL: ps_mesa_i16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v0.l
; GFX1250-TRUE16-NEXT: global_store_b16 v[0:1], v0, off
@@ -1331,6 +1393,8 @@ define amdgpu_ps void @ps_mesa_i16(i16 %arg0) {
;
; GFX1250-FAKE16-LABEL: ps_mesa_i16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_add_nc_u16 v0, v0, v0
; GFX1250-FAKE16-NEXT: global_store_b16 v[0:1], v0, off
@@ -1367,6 +1431,8 @@ define amdgpu_ps void @ps_mesa_inreg_i16(i16 inreg %arg0) {
;
; GFX1250-LABEL: ps_mesa_inreg_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_co_i32 s0, s0, s0
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1396,6 +1462,8 @@ define amdgpu_ps i16 @ret_ps_mesa_i16() {
;
; GFX1250-LABEL: ret_ps_mesa_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_movk_i32 s0, 0x7b
; GFX1250-NEXT: ; return to shader part epilog
@@ -1435,6 +1503,8 @@ define amdgpu_kernel void @amd_kernel_i8(i8 %arg0) {
;
; GFX1250-LABEL: amd_kernel_i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1501,6 +1571,8 @@ define amdgpu_kernel void @amd_kernel_v2i8(<2 x i8> %arg0) {
;
; GFX1250-LABEL: amd_kernel_v2i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 0
@@ -1603,6 +1675,8 @@ define amdgpu_kernel void @amd_kernel_v4i8(<4 x i8> %arg0) {
;
; GFX1250-LABEL: amd_kernel_v4i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 0
@@ -1702,6 +1776,8 @@ define amdgpu_kernel void @amd_kernel_v3i8(<3 x i8> %arg0) {
;
; GFX1250-LABEL: amd_kernel_v3i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 2
@@ -1823,6 +1899,8 @@ define amdgpu_kernel void @amd_kernel_v5i8(<5 x i8> %arg0) {
;
; GFX1250-LABEL: amd_kernel_v5i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4
@@ -1989,6 +2067,8 @@ define amdgpu_kernel void @amd_kernel_v8i8(<8 x i8> %arg0) {
;
; GFX1250-LABEL: amd_kernel_v8i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 0
@@ -2269,6 +2349,8 @@ define amdgpu_kernel void @amd_kernel_v16i8(<16 x i8> %arg0) {
;
; GFX1250-LABEL: amd_kernel_v16i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[4:5], 0
@@ -2792,6 +2874,8 @@ define amdgpu_kernel void @amd_kernel_v32i8(<32 x i8> %arg0) {
;
; GFX1250-LABEL: amd_kernel_v32i8:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[8:9], 16
@@ -2962,6 +3046,8 @@ define amdgpu_cs void @amdgpu_cs_i1(i1 %arg0) {
;
; GFX1250-LABEL: amdgpu_cs_i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-NEXT: global_store_b8 v[0:1], v0, off
@@ -3087,6 +3173,8 @@ define amdgpu_cs void @amdgpu_cs_v8i1(<8 x i1> %arg0) {
;
; GFX1250-TRUE16-LABEL: amdgpu_cs_v8i1:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v6.l, 1
@@ -3113,6 +3201,8 @@ define amdgpu_cs void @amdgpu_cs_v8i1(<8 x i1> %arg0) {
;
; GFX1250-FAKE16-LABEL: amdgpu_cs_v8i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v6, 1, v6
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 1, v2
@@ -3354,6 +3444,8 @@ define amdgpu_cs void @amdgpu_cs_v16i1(<16 x i1> %arg0) {
;
; GFX1250-TRUE16-LABEL: amdgpu_cs_v16i1:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v6.l, 1
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 3, v7.l
@@ -3399,6 +3491,8 @@ define amdgpu_cs void @amdgpu_cs_v16i1(<16 x i1> %arg0) {
;
; GFX1250-FAKE16-LABEL: amdgpu_cs_v16i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v6, 1, v6
; GFX1250-FAKE16-NEXT: v_lshlrev_b16 v7, 3, v7
@@ -3838,6 +3932,8 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
;
; GFX1250-TRUE16-LABEL: amdgpu_cs_v32i1:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v18.h, 3, v19.l
; GFX1250-TRUE16-NEXT: v_and_b16 v19.l, v22.l, 1
@@ -3916,6 +4012,8 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
;
; GFX1250-FAKE16-LABEL: amdgpu_cs_v32i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v18, 1, v18
; GFX1250-FAKE16-NEXT: v_lshlrev_b16 v19, 3, v19
@@ -4025,6 +4123,8 @@ define amdgpu_cs void @amdgpu_cs_inreg_i1(i1 inreg %arg0) {
;
; GFX1250-LABEL: amdgpu_cs_inreg_i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s0, 1
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -4122,6 +4222,8 @@ define amdgpu_cs void @amdgpu_cs_inreg_v8i1(<8 x i1> inreg %arg0) {
;
; GFX1250-LABEL: amdgpu_cs_inreg_v8i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s6, s6, 1
; GFX1250-NEXT: s_lshl_b32 s5, s5, 1
@@ -4311,6 +4413,8 @@ define amdgpu_cs void @amdgpu_cs_inreg_v16i1(<16 x i1> inreg %arg0) {
;
; GFX1250-LABEL: amdgpu_cs_inreg_v16i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s10, s10, 1
; GFX1250-NEXT: s_lshl_b32 s9, s9, 1
@@ -4668,6 +4772,8 @@ define amdgpu_cs void @amdgpu_cs_inreg_v32i1(<32 x i1> inreg %arg0) {
;
; GFX1250-LABEL: amdgpu_cs_inreg_v32i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s10, s10, 1
; GFX1250-NEXT: s_lshl_b32 s9, s9, 1
@@ -4793,6 +4899,8 @@ define amdgpu_cs void @amdgpu_cs_i1_sext(i1 signext %arg0) {
;
; GFX1250-LABEL: amdgpu_cs_i1_sext:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-NEXT: global_store_b8 v[0:1], v0, off
@@ -4821,6 +4929,8 @@ define amdgpu_cs void @amdgpu_cs_i1_zext(i1 zeroext %arg0) {
;
; GFX1250-LABEL: amdgpu_cs_i1_zext:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_store_b8 v[0:1], v0, off
; GFX1250-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index a6d6abe6493c1..aa069acd43fe2 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -115,6 +115,8 @@ define amdgpu_kernel void @sadd64rr(ptr addrspace(1) %out, i64 %a, i64 %b) {
;
; GFX1250-LABEL: sadd64rr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -261,6 +263,8 @@ define amdgpu_kernel void @sadd64ri(ptr addrspace(1) %out, i64 %a) {
;
; GFX1250-LABEL: sadd64ri:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -393,6 +397,8 @@ define amdgpu_kernel void @vadd64rr(ptr addrspace(1) %out, i64 %a) {
;
; GFX1250-LABEL: vadd64rr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -525,6 +531,8 @@ define amdgpu_kernel void @vadd64ri(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: vadd64ri:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -660,6 +668,8 @@ define amdgpu_kernel void @suaddo32(ptr addrspace(1) %out, ptr addrspace(1) %car
;
; GFX1250-LABEL: suaddo32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
@@ -831,6 +841,8 @@ define amdgpu_kernel void @uaddo32_vcc_user(ptr addrspace(1) %out, ptr addrspace
;
; GFX1250-LABEL: uaddo32_vcc_user:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
@@ -1033,6 +1045,8 @@ define amdgpu_kernel void @suaddo64(ptr addrspace(1) %out, ptr addrspace(1) %car
;
; GFX1250-LABEL: suaddo64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1229,6 +1243,8 @@ define amdgpu_kernel void @vuaddo64(ptr addrspace(1) %out, ptr addrspace(1) %car
;
; GFX1250-LABEL: vuaddo64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
@@ -1414,6 +1430,8 @@ define amdgpu_kernel void @ssub64rr(ptr addrspace(1) %out, i64 %a, i64 %b) {
;
; GFX1250-LABEL: ssub64rr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1560,6 +1578,8 @@ define amdgpu_kernel void @ssub64ri(ptr addrspace(1) %out, i64 %a) {
;
; GFX1250-LABEL: ssub64ri:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -1692,6 +1712,8 @@ define amdgpu_kernel void @vsub64rr(ptr addrspace(1) %out, i64 %a) {
;
; GFX1250-LABEL: vsub64rr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -1824,6 +1846,8 @@ define amdgpu_kernel void @vsub64ri(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: vsub64ri:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -1959,6 +1983,8 @@ define amdgpu_kernel void @susubo32(ptr addrspace(1) %out, ptr addrspace(1) %car
;
; GFX1250-LABEL: susubo32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
@@ -2131,6 +2157,8 @@ define amdgpu_kernel void @usubo32_vcc_user(ptr addrspace(1) %out, ptr addrspace
;
; GFX1250-LABEL: usubo32_vcc_user:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
@@ -2333,6 +2361,8 @@ define amdgpu_kernel void @susubo64(ptr addrspace(1) %out, ptr addrspace(1) %car
;
; GFX1250-LABEL: susubo64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2529,6 +2559,8 @@ define amdgpu_kernel void @vusubo64(ptr addrspace(1) %out, ptr addrspace(1) %car
;
; GFX1250-LABEL: vusubo64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
@@ -3726,6 +3758,8 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
;
; GFX1250-LABEL: sudiv64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
diff --git a/llvm/test/CodeGen/AMDGPU/clmul.ll b/llvm/test/CodeGen/AMDGPU/clmul.ll
index fea2ff0681a88..cdae4ee86c974 100644
--- a/llvm/test/CodeGen/AMDGPU/clmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/clmul.ll
@@ -389,6 +389,8 @@ define amdgpu_kernel void @test_clmul_i32(ptr addrspace(1) %out, ptr addrspace(1
;
; GFX1250-LABEL: test_clmul_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -1761,6 +1763,8 @@ define amdgpu_kernel void @test_clmulr_i32(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1250-LABEL: test_clmulr_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -3233,6 +3237,8 @@ define amdgpu_kernel void @test_clmulh_i32(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1250-LABEL: test_clmulh_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
diff --git a/llvm/test/CodeGen/AMDGPU/code-size-estimate.ll b/llvm/test/CodeGen/AMDGPU/code-size-estimate.ll
index b59bcb7967af2..617c1ad6dbe99 100644
--- a/llvm/test/CodeGen/AMDGPU/code-size-estimate.ll
+++ b/llvm/test/CodeGen/AMDGPU/code-size-estimate.ll
@@ -581,6 +581,8 @@ define amdgpu_ps float @s_fmaak_f32(float inreg %x, float inreg %y) {
;
; GFX1250-LABEL: s_fmaak_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb ; encoding: [0x7c,0x00,0x0b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+; GFX1250-NEXT: v_nop ; encoding: [0x00,0x00,0x00,0x7e]
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: ; encoding: [0x41,0x06,0x80,0xb9,0x01,0x00,0x00,0x00]
; GFX1250-NEXT: s_fmaak_f32 s0, s0, s1, 0x43800000 ; encoding: [0x00,0x01,0x80,0xa2,0x00,0x00,0x80,0x43]
@@ -596,7 +598,7 @@ define amdgpu_ps float @s_fmaak_f32(float inreg %x, float inreg %y) {
; GFX1100: codeLenInByte = 16
; GFX1150: codeLenInByte = 16
; GFX1200: codeLenInByte = 16
-; GFX1250: codeLenInByte = 24
+; GFX1250: codeLenInByte = 40
define double @v_mul_f64_vop2_literal_32(double %x) {
; GFX9-LABEL: v_mul_f64_vop2_literal_32:
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index 00136d8676bac..4692adcfb08ec 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -6,6 +6,8 @@
define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out, i1 %br0, i32 %delta) local_unnamed_addr #0 {
; COEXEC-LABEL: ds_wmma:
; COEXEC: ; %bb.0: ; %entry
+; COEXEC-NEXT: global_wb
+; COEXEC-NEXT: v_nop
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: s_clause 0x1
@@ -91,6 +93,8 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
;
; GCN-LABEL: ds_wmma:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
@@ -247,6 +251,8 @@ end:
define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace(3) %base1, ptr addrspace(1) %out, i1 %br0, i32 %delta) local_unnamed_addr #0 {
; COEXEC-LABEL: ds_wmma_permute:
; COEXEC: ; %bb.0: ; %entry
+; COEXEC-NEXT: global_wb
+; COEXEC-NEXT: v_nop
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: s_mov_b32 s6, 0
; COEXEC-NEXT: s_clause 0x1
@@ -363,6 +369,8 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
;
; GCN-LABEL: ds_wmma_permute:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
index a895424bc2803..8c28e037cdb3c 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
@@ -10,6 +10,8 @@
define amdgpu_kernel void @simple_read2_f32(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -36,6 +38,8 @@ define amdgpu_kernel void @simple_read2_f32(ptr addrspace(1) %out) #0 {
define amdgpu_kernel void @simple_read2_f32_max_offset(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32_max_offset:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -62,6 +66,8 @@ define amdgpu_kernel void @simple_read2_f32_max_offset(ptr addrspace(1) %out) #0
define amdgpu_kernel void @simple_read2_f32_too_far(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32_too_far:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -89,6 +95,8 @@ define amdgpu_kernel void @simple_read2_f32_too_far(ptr addrspace(1) %out) #0 {
define amdgpu_kernel void @simple_read2_f32_x2(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32_x2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -131,6 +139,8 @@ define amdgpu_kernel void @simple_read2_f32_x2(ptr addrspace(1) %out) #0 {
define amdgpu_kernel void @simple_read2_f32_x2_barrier(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32_x2_barrier:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -179,6 +189,8 @@ define amdgpu_kernel void @simple_read2_f32_x2_barrier(ptr addrspace(1) %out) #0
define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32_x2_nonzero_base:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -225,6 +237,8 @@ define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(ptr addrspace(1) %ou
define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(ptr addrspace(1) %out, <2 x ptr addrspace(3)> %lds.ptr) #0 {
; GFX1250-LABEL: read2_ptr_is_subreg_arg_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -258,6 +272,8 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(ptr addrspace(1) %out, <2
define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(ptr addrspace(1) %out, <2 x ptr addrspace(3)> %lds.ptr) #0 {
; GFX1250-LABEL: read2_ptr_is_subreg_arg_offset_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -291,6 +307,8 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(ptr addrspace(1) %
define amdgpu_kernel void @read2_ptr_is_subreg_f32(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: read2_ptr_is_subreg_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -323,6 +341,8 @@ define amdgpu_kernel void @read2_ptr_is_subreg_f32(ptr addrspace(1) %out) #0 {
define amdgpu_kernel void @simple_read2_f32_volatile_0(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32_volatile_0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -350,6 +370,8 @@ define amdgpu_kernel void @simple_read2_f32_volatile_0(ptr addrspace(1) %out) #0
define amdgpu_kernel void @simple_read2_f32_volatile_1(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f32_volatile_1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -378,6 +400,8 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(ptr addrspace(1) %out) #0
define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; GFX1250-LABEL: unaligned_read2_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -405,6 +429,8 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; GFX1250-LABEL: unaligned_offset_read2_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -432,6 +458,8 @@ define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; GFX1250-LABEL: misaligned_2_simple_read2_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -459,6 +487,8 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out,
define amdgpu_kernel void @simple_read2_f64(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -486,6 +516,8 @@ define amdgpu_kernel void @simple_read2_f64(ptr addrspace(1) %out) #0 {
define amdgpu_kernel void @simple_read2_f64_max_offset(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f64_max_offset:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -513,6 +545,8 @@ define amdgpu_kernel void @simple_read2_f64_max_offset(ptr addrspace(1) %out) #0
define amdgpu_kernel void @simple_read2_f64_too_far(ptr addrspace(1) %out) #0 {
; GFX1250-LABEL: simple_read2_f64_too_far:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -542,6 +576,8 @@ define amdgpu_kernel void @simple_read2_f64_too_far(ptr addrspace(1) %out) #0 {
define amdgpu_kernel void @misaligned_read2_f64(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; GFX1250-LABEL: misaligned_read2_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
@@ -573,6 +609,8 @@ define amdgpu_kernel void @misaligned_read2_f64(ptr addrspace(1) %out, ptr addrs
define amdgpu_kernel void @load_constant_adjacent_offsets(ptr addrspace(1) %out) {
; GFX1250-LABEL: load_constant_adjacent_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -592,6 +630,8 @@ define amdgpu_kernel void @load_constant_adjacent_offsets(ptr addrspace(1) %out)
define amdgpu_kernel void @load_constant_disjoint_offsets(ptr addrspace(1) %out) {
; GFX1250-LABEL: load_constant_disjoint_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -613,6 +653,8 @@ define amdgpu_kernel void @load_constant_disjoint_offsets(ptr addrspace(1) %out)
define amdgpu_kernel void @load_misaligned64_constant_offsets(ptr addrspace(1) %out) {
; GFX1250-LABEL: load_misaligned64_constant_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -634,6 +676,8 @@ define amdgpu_kernel void @load_misaligned64_constant_offsets(ptr addrspace(1) %
define amdgpu_kernel void @load_misaligned64_constant_large_offsets(ptr addrspace(1) %out) {
; GFX1250-LABEL: load_misaligned64_constant_large_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -657,6 +701,8 @@ define amdgpu_kernel void @load_misaligned64_constant_large_offsets(ptr addrspac
define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(ptr addrspace(1) %C, i32 %lda, i32 %ldb) #0 {
; GFX1250-LABEL: sgemm_inner_loop_read2_sequence:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_bfe_u32 s0, ttmp6, 0x4000c
; GFX1250-NEXT: s_and_b32 s1, ttmp6, 15
@@ -746,6 +792,8 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(ptr addrspace(1) %C,
define amdgpu_kernel void @misaligned_read2_v2i32(ptr addrspace(1) %out, ptr addrspace(3) %in) #0 {
; GFX1250-LABEL: misaligned_read2_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -762,6 +810,8 @@ define amdgpu_kernel void @misaligned_read2_v2i32(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @misaligned_read2_i64(ptr addrspace(1) %out, ptr addrspace(3) %in) #0 {
; GFX1250-LABEL: misaligned_read2_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -778,6 +828,8 @@ define amdgpu_kernel void @misaligned_read2_i64(ptr addrspace(1) %out, ptr addrs
define amdgpu_kernel void @ds_read_diff_base_interleaving(
; GFX1250-LABEL: ds_read_diff_base_interleaving:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 6, v0 :: v_dual_lshlrev_b32 v0, 2, v0
@@ -845,6 +897,8 @@ bb:
define amdgpu_kernel void @ds_read_call_read(ptr addrspace(1) %out, ptr addrspace(3) %arg) {
; GFX1250-LABEL: ds_read_call_read:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s32, 0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[36:38], s[4:5], 0x0 nv
@@ -881,6 +935,8 @@ define amdgpu_kernel void @ds_read_call_read(ptr addrspace(1) %out, ptr addrspac
define amdgpu_kernel void @read2_v2i32_align1_odd_offset(ptr addrspace(1) %out) {
; GFX1250-LABEL: read2_v2i32_align1_odd_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
index 48d7444401e4a..f05d248a52b15 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
@@ -34,6 +34,8 @@ define amdgpu_kernel void @simple_write2_one_val_f32(ptr addrspace(1) %C, ptr ad
;
; GFX1250-LABEL: simple_write2_one_val_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -86,6 +88,8 @@ define amdgpu_kernel void @simple_write2_two_val_f32(ptr addrspace(1) %C, ptr ad
;
; GFX1250-LABEL: simple_write2_two_val_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -147,6 +151,8 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(ptr addrspace(1)
;
; GFX1250-LABEL: simple_write2_two_val_f32_volatile_0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -209,6 +215,8 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(ptr addrspace(1)
;
; GFX1250-LABEL: simple_write2_two_val_f32_volatile_1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -275,6 +283,8 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(ptr addrspace
;
; GFX1250-LABEL: simple_write2_two_val_subreg2_mixed_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -330,6 +340,8 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(ptr addrspace(1) %C
;
; GFX1250-LABEL: simple_write2_two_val_subreg2_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -382,6 +394,8 @@ define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(ptr addrspace(1) %C
;
; GFX1250-LABEL: simple_write2_two_val_subreg4_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -435,6 +449,8 @@ define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(ptr addrspace(1)
;
; GFX1250-LABEL: simple_write2_two_val_max_offset_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -496,6 +512,8 @@ define amdgpu_kernel void @simple_write2_two_val_too_far_f32(ptr addrspace(1) %C
;
; GFX1250-LABEL: simple_write2_two_val_too_far_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -557,6 +575,8 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2(ptr addrspace(1) %C, ptr
;
; GFX1250-LABEL: simple_write2_two_val_f32_x2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -629,6 +649,8 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(ptr addrspa
;
; GFX1250-LABEL: simple_write2_two_val_f32_x2_nonzero_base:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
@@ -709,6 +731,8 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(ptr addrspace(1) %C
;
; GFX1250-LABEL: write2_ptr_subreg_arg_two_val_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -773,6 +797,8 @@ define amdgpu_kernel void @simple_write2_one_val_f64(ptr addrspace(1) %C, ptr ad
;
; GFX1250-LABEL: simple_write2_one_val_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
@@ -827,6 +853,8 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(ptr addrspace(1)
;
; GFX1250-LABEL: misaligned_simple_write2_one_val_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
@@ -927,6 +955,8 @@ define amdgpu_kernel void @unaligned_offset_simple_write2_one_val_f64(ptr addrsp
;
; GFX1250-LABEL: unaligned_offset_simple_write2_one_val_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
@@ -982,6 +1012,8 @@ define amdgpu_kernel void @simple_write2_two_val_f64(ptr addrspace(1) %C, ptr ad
;
; GFX1250-LABEL: simple_write2_two_val_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
@@ -1029,6 +1061,8 @@ define amdgpu_kernel void @store_constant_adjacent_offsets() {
;
; GFX1250-LABEL: store_constant_adjacent_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 0x7b0000007b
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -1057,6 +1091,8 @@ define amdgpu_kernel void @store_constant_disjoint_offsets() {
;
; GFX1250-LABEL: store_constant_disjoint_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0x7b :: v_dual_mov_b32 v1, 0
; GFX1250-NEXT: ds_store_2addr_b32 v1, v0, v0 offset1:2
@@ -1090,6 +1126,8 @@ define amdgpu_kernel void @store_misaligned64_constant_offsets() {
;
; GFX1250-LABEL: store_misaligned64_constant_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0x7b :: v_dual_mov_b32 v1, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1127,6 +1165,8 @@ define amdgpu_kernel void @store_misaligned64_constant_large_offsets() {
;
; GFX1250-LABEL: store_misaligned64_constant_large_offsets:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 0x7b
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -1187,6 +1227,8 @@ define amdgpu_kernel void @write2_sgemm_sequence(ptr addrspace(1) %C, i32 %lda,
;
; GFX1250-LABEL: write2_sgemm_sequence:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; GFX1250-NEXT: s_and_b32 s2, ttmp6, 15
@@ -1304,6 +1346,8 @@ define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(ptr addrspace(3)
;
; GFX1250-LABEL: simple_write2_v4f32_superreg_align4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x8 nv
@@ -1373,6 +1417,8 @@ define amdgpu_kernel void @write2_v2i32_align1_odd_offset() {
;
; GFX1250-LABEL: write2_v2i32_align1_odd_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 0x1c80000007b
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index 1b0a5217fbb2b..68d9cdf4730f3 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -19,6 +19,8 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
define amdgpu_kernel void @test_fold_canonicalize_undef_value_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_undef_value_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -28,6 +30,8 @@ define amdgpu_kernel void @test_fold_canonicalize_undef_value_bf16(ptr addrspace
;
; REAL16-LABEL: test_fold_canonicalize_undef_value_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -43,6 +47,8 @@ define amdgpu_kernel void @test_fold_canonicalize_undef_value_bf16(ptr addrspace
define amdgpu_kernel void @v_test_canonicalize_var_bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: v_test_canonicalize_var_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -64,6 +70,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_bf16(ptr addrspace(1) %out) #
define amdgpu_kernel void @s_test_canonicalize_var_bf16(ptr addrspace(1) %out, i16 zeroext %val.arg) #1 {
; GFX1250-LABEL: s_test_canonicalize_var_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -108,6 +116,8 @@ define <2 x bfloat> @v_test_canonicalize_build_vector_v2bf16(bfloat %lo, bfloat
define amdgpu_kernel void @v_test_canonicalize_fabs_var_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: v_test_canonicalize_fabs_var_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -125,6 +135,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_bf16(ptr addrspace(1) %o
;
; REAL16-LABEL: v_test_canonicalize_fabs_var_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -150,6 +162,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_bf16(ptr addrspace(1) %o
define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: v_test_canonicalize_fneg_fabs_var_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -167,6 +181,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_bf16(ptr addrspace(
;
; REAL16-LABEL: v_test_canonicalize_fneg_fabs_var_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -192,6 +208,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_bf16(ptr addrspace(
define amdgpu_kernel void @v_test_canonicalize_fneg_var_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: v_test_canonicalize_fneg_var_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -209,6 +227,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_bf16(ptr addrspace(1) %o
;
; REAL16-LABEL: v_test_canonicalize_fneg_var_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -233,6 +253,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_bf16(ptr addrspace(1) %o
define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_bf16(ptr addrspace(1) %out) #2 {
; FAKE16-LABEL: v_test_no_denormals_canonicalize_fneg_var_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -250,6 +272,8 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_bf16(ptr ad
;
; REAL16-LABEL: v_test_no_denormals_canonicalize_fneg_var_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -274,6 +298,8 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_bf16(ptr ad
define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_bf16(ptr addrspace(1) %out) #2 {
; FAKE16-LABEL: v_test_no_denormals_canonicalize_fneg_fabs_var_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -291,6 +317,8 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_bf16(p
;
; REAL16-LABEL: v_test_no_denormals_canonicalize_fneg_fabs_var_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -316,6 +344,8 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_bf16(p
define amdgpu_kernel void @test_fold_canonicalize_p0_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_p0_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -325,6 +355,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_bf16(ptr addrspace(1) %out)
;
; REAL16-LABEL: test_fold_canonicalize_p0_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -340,6 +372,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_bf16(ptr addrspace(1) %out)
define amdgpu_kernel void @test_fold_canonicalize_n0_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_n0_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xffff8000
@@ -349,6 +383,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_bf16(ptr addrspace(1) %out)
;
; REAL16-LABEL: test_fold_canonicalize_n0_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -364,6 +400,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_bf16(ptr addrspace(1) %out)
define amdgpu_kernel void @test_fold_canonicalize_p1_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_p1_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3f80
@@ -373,6 +411,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_bf16(ptr addrspace(1) %out)
;
; REAL16-LABEL: test_fold_canonicalize_p1_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -388,6 +428,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_bf16(ptr addrspace(1) %out)
define amdgpu_kernel void @test_fold_canonicalize_n1_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_n1_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xffffbf80
@@ -397,6 +439,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_bf16(ptr addrspace(1) %out)
;
; REAL16-LABEL: test_fold_canonicalize_n1_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -412,6 +456,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_bf16(ptr addrspace(1) %out)
define amdgpu_kernel void @test_fold_canonicalize_literal_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_literal_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x4180
@@ -421,6 +467,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_bf16(ptr addrspace(1)
;
; REAL16-LABEL: test_fold_canonicalize_literal_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -436,6 +484,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_bf16(ptr addrspace(1)
define amdgpu_kernel void @test_default_denormals_fold_canonicalize_denormal0_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_default_denormals_fold_canonicalize_denormal0_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3ff
@@ -445,6 +495,8 @@ define amdgpu_kernel void @test_default_denormals_fold_canonicalize_denormal0_bf
;
; REAL16-LABEL: test_default_denormals_fold_canonicalize_denormal0_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -460,6 +512,8 @@ define amdgpu_kernel void @test_default_denormals_fold_canonicalize_denormal0_bf
define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_bf16(ptr addrspace(1) %out) #3 {
; FAKE16-LABEL: test_denormals_fold_canonicalize_denormal0_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3ff
@@ -469,6 +523,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_bf16(ptr a
;
; REAL16-LABEL: test_denormals_fold_canonicalize_denormal0_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -484,6 +540,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_bf16(ptr a
define amdgpu_kernel void @test_default_denormals_fold_canonicalize_denormal1_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_default_denormals_fold_canonicalize_denormal1_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xffff83ff
@@ -493,6 +551,8 @@ define amdgpu_kernel void @test_default_denormals_fold_canonicalize_denormal1_bf
;
; REAL16-LABEL: test_default_denormals_fold_canonicalize_denormal1_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -508,6 +568,8 @@ define amdgpu_kernel void @test_default_denormals_fold_canonicalize_denormal1_bf
define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_bf16(ptr addrspace(1) %out) #3 {
; FAKE16-LABEL: test_denormals_fold_canonicalize_denormal1_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xffff83ff
@@ -517,6 +579,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_bf16(ptr a
;
; REAL16-LABEL: test_denormals_fold_canonicalize_denormal1_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -532,6 +596,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_bf16(ptr a
define amdgpu_kernel void @test_fold_canonicalize_qnan_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_qnan_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7c00
@@ -541,6 +607,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_bf16(ptr addrspace(1) %ou
;
; REAL16-LABEL: test_fold_canonicalize_qnan_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -556,6 +624,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_bf16(ptr addrspace(1) %ou
define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg1_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc0
@@ -565,6 +635,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_bf16(ptr addrs
;
; REAL16-LABEL: test_fold_canonicalize_qnan_value_neg1_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -580,6 +652,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_bf16(ptr addrs
define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg2_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc0
@@ -589,6 +663,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_bf16(ptr addrs
;
; REAL16-LABEL: test_fold_canonicalize_qnan_value_neg2_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -604,6 +680,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_bf16(ptr addrs
define amdgpu_kernel void @test_fold_canonicalize_snan0_value_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_snan0_value_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7c01
@@ -613,6 +691,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_bf16(ptr addrspace
;
; REAL16-LABEL: test_fold_canonicalize_snan0_value_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -628,6 +708,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_bf16(ptr addrspace
define amdgpu_kernel void @test_fold_canonicalize_snan1_value_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_snan1_value_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7dff
@@ -637,6 +719,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_bf16(ptr addrspace
;
; REAL16-LABEL: test_fold_canonicalize_snan1_value_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -652,6 +736,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_bf16(ptr addrspace
define amdgpu_kernel void @test_fold_canonicalize_snan2_value_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_snan2_value_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xfffffdff
@@ -661,6 +747,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_bf16(ptr addrspace
;
; REAL16-LABEL: test_fold_canonicalize_snan2_value_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -676,6 +764,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_bf16(ptr addrspace
define amdgpu_kernel void @test_fold_canonicalize_snan3_value_bf16(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: test_fold_canonicalize_snan3_value_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xfffffc01
@@ -685,6 +775,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_bf16(ptr addrspace
;
; REAL16-LABEL: test_fold_canonicalize_snan3_value_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -700,6 +792,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_bf16(ptr addrspace
define amdgpu_kernel void @v_test_canonicalize_var_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: v_test_canonicalize_var_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -721,6 +815,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2bf16(ptr addrspace(1) %out)
define amdgpu_kernel void @v_test_canonicalize_fabs_var_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: v_test_canonicalize_fabs_var_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -745,6 +841,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_v2bf16(ptr addrspace(1)
define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: v_test_canonicalize_fneg_fabs_var_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -770,6 +868,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2bf16(ptr addrspac
define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: v_test_canonicalize_fneg_var_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -792,6 +892,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2bf16(ptr addrspace(1)
define amdgpu_kernel void @s_test_canonicalize_var_v2bf16(ptr addrspace(1) %out, i32 zeroext %val.arg) #1 {
; GFX1250-LABEL: s_test_canonicalize_var_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -808,6 +910,8 @@ define amdgpu_kernel void @s_test_canonicalize_var_v2bf16(ptr addrspace(1) %out,
define amdgpu_kernel void @test_fold_canonicalize_p0_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_p0_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -822,6 +926,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_v2bf16(ptr addrspace(1) %ou
define amdgpu_kernel void @test_fold_canonicalize_n0_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_n0_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x80008000
@@ -836,6 +942,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_v2bf16(ptr addrspace(1) %ou
define amdgpu_kernel void @test_fold_canonicalize_p1_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_p1_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3f803f80
@@ -850,6 +958,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_v2bf16(ptr addrspace(1) %ou
define amdgpu_kernel void @test_fold_canonicalize_n1_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_n1_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xbf80bf80
@@ -864,6 +974,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_v2bf16(ptr addrspace(1) %ou
define amdgpu_kernel void @test_fold_canonicalize_literal_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_literal_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x41804180
@@ -878,6 +990,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_v2bf16(ptr addrspace(1
define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_no_denormals_fold_canonicalize_denormal0_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3ff03ff
@@ -892,6 +1006,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_v2bf16(
define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_v2bf16(ptr addrspace(1) %out) #3 {
; GFX1250-LABEL: test_denormals_fold_canonicalize_denormal0_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3ff03ff
@@ -906,6 +1022,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_v2bf16(ptr
define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_no_denormals_fold_canonicalize_denormal1_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x83ff83ff
@@ -920,6 +1038,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_v2bf16(
define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_v2bf16(ptr addrspace(1) %out) #3 {
; GFX1250-LABEL: test_denormals_fold_canonicalize_denormal1_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x83ff83ff
@@ -934,6 +1054,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_v2bf16(ptr
define amdgpu_kernel void @test_fold_canonicalize_qnan_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_qnan_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7c007c00
@@ -948,6 +1070,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_v2bf16(ptr addrspace(1) %
define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_qnan_value_neg1_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc07fc0
@@ -962,6 +1086,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_v2bf16(ptr add
define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_qnan_value_neg2_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc07fc0
@@ -976,6 +1102,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_v2bf16(ptr add
define amdgpu_kernel void @test_fold_canonicalize_snan0_value_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_snan0_value_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7c017c01
@@ -990,6 +1118,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_v2bf16(ptr addrspa
define amdgpu_kernel void @test_fold_canonicalize_snan1_value_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_snan1_value_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7dff7dff
@@ -1004,6 +1134,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_v2bf16(ptr addrspa
define amdgpu_kernel void @test_fold_canonicalize_snan2_value_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_snan2_value_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xfdfffdff
@@ -1018,6 +1150,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_v2bf16(ptr addrspa
define amdgpu_kernel void @test_fold_canonicalize_snan3_value_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: test_fold_canonicalize_snan3_value_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0xfc01fc01
@@ -1056,6 +1190,8 @@ define <4 x bfloat> @v_test_canonicalize_var_v4bf16(<4 x bfloat> %val) #1 {
define amdgpu_kernel void @s_test_canonicalize_undef_v2bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: s_test_canonicalize_undef_v2bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -1205,6 +1341,8 @@ define <2 x bfloat> @v_test_canonicalize_k_reg_v2bf16(bfloat %val) #1 {
define amdgpu_kernel void @s_test_canonicalize_undef_v4bf16(ptr addrspace(1) %out) #1 {
; GFX1250-LABEL: s_test_canonicalize_undef_v4bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index 283af41020043..7763da0bedf79 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -132,6 +132,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f32(ptr addrspace(1) %out) #1
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_var_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -144,6 +146,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f32(ptr addrspace(1) %out) #1
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_var_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -156,6 +160,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f32(ptr addrspace(1) %out) #1
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_var_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -168,6 +174,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f32(ptr addrspace(1) %out) #1
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_var_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -302,6 +310,8 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
;
; GFX1251-SDAG-TRUE16-LABEL: s_test_canonicalize_var_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -312,6 +322,8 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
;
; GFX1251-GISEL-TRUE16-LABEL: s_test_canonicalize_var_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -322,6 +334,8 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
;
; GFX1251-SDAG-FAKE16-LABEL: s_test_canonicalize_var_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -332,6 +346,8 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
;
; GFX1251-GISEL-FAKE16-LABEL: s_test_canonicalize_var_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -443,6 +459,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f32(ptr addrspace(1) %ou
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_fabs_var_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -455,6 +473,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f32(ptr addrspace(1) %ou
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_fabs_var_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -467,6 +487,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f32(ptr addrspace(1) %ou
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_fabs_var_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -479,6 +501,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f32(ptr addrspace(1) %ou
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_fabs_var_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -594,6 +618,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f32(ptr addrspace(1
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_fneg_fabs_var_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -606,6 +632,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f32(ptr addrspace(1
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_fneg_fabs_var_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -618,6 +646,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f32(ptr addrspace(1
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_fneg_fabs_var_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -630,6 +660,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f32(ptr addrspace(1
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_fneg_fabs_var_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -746,6 +778,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f32(ptr addrspace(1) %ou
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_fneg_var_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -758,6 +792,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f32(ptr addrspace(1) %ou
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_fneg_var_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -770,6 +806,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f32(ptr addrspace(1) %ou
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_fneg_var_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -782,6 +820,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f32(ptr addrspace(1) %ou
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_fneg_var_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -838,6 +878,8 @@ define amdgpu_kernel void @test_fold_canonicalize_undef_f32(ptr addrspace(1) %ou
;
; GFX1251-LABEL: test_fold_canonicalize_undef_f32:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-NEXT: v_mov_b32_e32 v0, 0
@@ -889,6 +931,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f32(ptr addrspace(1) %out)
;
; GFX1251-LABEL: test_fold_canonicalize_p0_f32:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-NEXT: v_mov_b32_e32 v0, 0
@@ -970,6 +1014,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f32(ptr addrspace(1) %out)
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_n0_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -980,6 +1026,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f32(ptr addrspace(1) %out)
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_n0_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_bfrev_b32_e32 v0, 1
@@ -990,6 +1038,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f32(ptr addrspace(1) %out)
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_n0_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1000,6 +1050,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f32(ptr addrspace(1) %out)
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_n0_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_bfrev_b32_e32 v0, 1
@@ -1078,6 +1130,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f32(ptr addrspace(1) %out)
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_p1_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 1.0
@@ -1087,6 +1141,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f32(ptr addrspace(1) %out)
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_p1_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 0
@@ -1096,6 +1152,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f32(ptr addrspace(1) %out)
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_p1_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 1.0
@@ -1105,6 +1163,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f32(ptr addrspace(1) %out)
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_p1_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, 1.0 :: v_dual_mov_b32 v1, 0
@@ -1182,6 +1242,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f32(ptr addrspace(1) %out)
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_n1_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, -1.0
@@ -1191,6 +1253,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f32(ptr addrspace(1) %out)
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_n1_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, -1.0 :: v_dual_mov_b32 v1, 0
@@ -1200,6 +1264,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f32(ptr addrspace(1) %out)
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_n1_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, -1.0
@@ -1209,6 +1275,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f32(ptr addrspace(1) %out)
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_n1_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, -1.0 :: v_dual_mov_b32 v1, 0
@@ -1286,6 +1354,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f32(ptr addrspace(1) %
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_literal_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x41800000
@@ -1295,6 +1365,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f32(ptr addrspace(1) %
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_literal_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, 0x41800000 :: v_dual_mov_b32 v1, 0
@@ -1304,6 +1376,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f32(ptr addrspace(1) %
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_literal_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x41800000
@@ -1313,6 +1387,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f32(ptr addrspace(1) %
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_literal_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, 0x41800000 :: v_dual_mov_b32 v1, 0
@@ -1364,6 +1440,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32(ptr
;
; GFX1251-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-NEXT: v_mov_b32_e32 v0, 0
@@ -1462,6 +1540,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-SDAG-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1472,6 +1552,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-GISEL-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_max_num_f32_e64 v0, 0x7fffff, 0x7fffff
@@ -1482,6 +1564,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-SDAG-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1492,6 +1576,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-GISEL-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_max_num_f32_e64 v0, 0x7fffff, 0x7fffff
@@ -1591,6 +1677,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-SDAG-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_out:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1601,6 +1689,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-GISEL-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_out:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_max_num_f32_e64 v0, 0x7fffff, 0x7fffff
@@ -1611,6 +1701,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-SDAG-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_out:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1621,6 +1713,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-GISEL-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_out:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_max_num_f32_e64 v0, 0x7fffff, 0x7fffff
@@ -1720,6 +1814,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-SDAG-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_in:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1730,6 +1826,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-GISEL-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_in:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_max_num_f32_e64 v0, 0x7fffff, 0x7fffff
@@ -1740,6 +1838,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-SDAG-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_in:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1750,6 +1850,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f32_dyn
;
; GFX1251-GISEL-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f32_dynamic_in:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_max_num_f32_e64 v0, 0x7fffff, 0x7fffff
@@ -1828,6 +1930,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f32(ptr ad
;
; GFX1251-SDAG-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal0_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fffff
@@ -1837,6 +1941,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f32(ptr ad
;
; GFX1251-GISEL-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal0_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, 0x7fffff :: v_dual_mov_b32 v1, 0
@@ -1846,6 +1952,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f32(ptr ad
;
; GFX1251-SDAG-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal0_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fffff
@@ -1855,6 +1963,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f32(ptr ad
;
; GFX1251-GISEL-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal0_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, 0x7fffff :: v_dual_mov_b32 v1, 0
@@ -1936,6 +2046,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f32(ptr
;
; GFX1251-SDAG-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1946,6 +2058,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f32(ptr
;
; GFX1251-GISEL-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_bfrev_b32_e32 v0, 1
@@ -1956,6 +2070,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f32(ptr
;
; GFX1251-SDAG-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -1966,6 +2082,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f32(ptr
;
; GFX1251-GISEL-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_bfrev_b32_e32 v0, 1
@@ -2044,6 +2162,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f32(ptr ad
;
; GFX1251-SDAG-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x807fffff
@@ -2053,6 +2173,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f32(ptr ad
;
; GFX1251-GISEL-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, 0x807fffff :: v_dual_mov_b32 v1, 0
@@ -2062,6 +2184,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f32(ptr ad
;
; GFX1251-SDAG-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x807fffff
@@ -2071,6 +2195,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f32(ptr ad
;
; GFX1251-GISEL-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal1_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, 0x807fffff :: v_dual_mov_b32 v1, 0
@@ -2148,6 +2274,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f32(ptr addrspace(1) %out
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_qnan_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2157,6 +2285,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f32(ptr addrspace(1) %out
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_qnan_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, 0x7fc00000 :: v_dual_mov_b32 v1, 0
@@ -2166,6 +2296,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f32(ptr addrspace(1) %out
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_qnan_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2175,6 +2307,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f32(ptr addrspace(1) %out
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_qnan_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, 0x7fc00000 :: v_dual_mov_b32 v1, 0
@@ -2265,6 +2399,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f32(ptr addrsp
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2274,6 +2410,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f32(ptr addrsp
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, -1 :: v_dual_mov_b32 v1, 0
@@ -2283,6 +2421,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f32(ptr addrsp
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2292,6 +2432,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f32(ptr addrsp
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, -1 :: v_dual_mov_b32 v1, 0
@@ -2382,6 +2524,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f32(ptr addrsp
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2391,6 +2535,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f32(ptr addrsp
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, -2 :: v_dual_mov_b32 v1, 0
@@ -2400,6 +2546,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f32(ptr addrsp
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2409,6 +2557,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f32(ptr addrsp
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_dual_mov_b32 v0, -2 :: v_dual_mov_b32 v1, 0
@@ -2503,6 +2653,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f32(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan0_value_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2512,6 +2664,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f32(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan0_value_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_max_num_f32_e64 v0, 0x7f800001, 0x7f800001
@@ -2522,6 +2676,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f32(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan0_value_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2531,6 +2687,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f32(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan0_value_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_max_num_f32_e64 v0, 0x7f800001, 0x7f800001
@@ -2626,6 +2784,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f32(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan1_value_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2635,6 +2795,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f32(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan1_value_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_max_num_f32_e64 v0, 0x7fbfffff, 0x7fbfffff
@@ -2645,6 +2807,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f32(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan1_value_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2654,6 +2818,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f32(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan1_value_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_max_num_f32_e64 v0, 0x7fbfffff, 0x7fbfffff
@@ -2749,6 +2915,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f32(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan2_value_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2758,6 +2926,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f32(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan2_value_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_max_num_f32_e64 v0, 0xff800001, 0xff800001
@@ -2768,6 +2938,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f32(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan2_value_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2777,6 +2949,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f32(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan2_value_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_max_num_f32_e64 v0, 0xff800001, 0xff800001
@@ -2872,6 +3046,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f32(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan3_value_f32:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2881,6 +3057,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f32(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan3_value_f32:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_max_num_f32_e64 v0, 0xffbfffff, 0xffbfffff
@@ -2891,6 +3069,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f32(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan3_value_f32:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc00000
@@ -2900,6 +3080,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f32(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan3_value_f32:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_max_num_f32_e64 v0, 0xffbfffff, 0xffbfffff
@@ -3011,6 +3193,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f64(ptr addrspace(1) %out) #1
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_var_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3023,6 +3207,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f64(ptr addrspace(1) %out) #1
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_var_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3035,6 +3221,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f64(ptr addrspace(1) %out) #1
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_var_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3047,6 +3235,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_f64(ptr addrspace(1) %out) #1
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_var_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3131,6 +3321,8 @@ define amdgpu_kernel void @s_test_canonicalize_var_f64(ptr addrspace(1) %out, do
;
; GFX1251-LABEL: s_test_canonicalize_var_f64:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-NEXT: v_mov_b32_e32 v2, 0
@@ -3242,6 +3434,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(ptr addrspace(1) %ou
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_fabs_var_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3254,6 +3448,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(ptr addrspace(1) %ou
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_fabs_var_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3266,6 +3462,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(ptr addrspace(1) %ou
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_fabs_var_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3278,6 +3476,8 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(ptr addrspace(1) %ou
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_fabs_var_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3393,6 +3593,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(ptr addrspace(1
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_fneg_fabs_var_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3405,6 +3607,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(ptr addrspace(1
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_fneg_fabs_var_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3417,6 +3621,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(ptr addrspace(1
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_fneg_fabs_var_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3429,6 +3635,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(ptr addrspace(1
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_fneg_fabs_var_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3545,6 +3753,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f64(ptr addrspace(1) %ou
;
; GFX1251-SDAG-TRUE16-LABEL: v_test_canonicalize_fneg_var_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3557,6 +3767,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f64(ptr addrspace(1) %ou
;
; GFX1251-GISEL-TRUE16-LABEL: v_test_canonicalize_fneg_var_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3569,6 +3781,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f64(ptr addrspace(1) %ou
;
; GFX1251-SDAG-FAKE16-LABEL: v_test_canonicalize_fneg_var_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3581,6 +3795,8 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_f64(ptr addrspace(1) %ou
;
; GFX1251-GISEL-FAKE16-LABEL: v_test_canonicalize_fneg_var_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3685,6 +3901,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_p0_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3695,6 +3913,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_p0_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0
@@ -3705,6 +3925,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_p0_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3715,6 +3937,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p0_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_p0_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0
@@ -3801,6 +4025,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_n0_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3811,6 +4037,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_n0_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0x8000000000000000
@@ -3821,6 +4049,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_n0_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3831,6 +4061,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n0_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_n0_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0x8000000000000000
@@ -3913,6 +4145,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_p1_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3923,6 +4157,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_p1_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 1.0
@@ -3933,6 +4169,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_p1_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -3943,6 +4181,8 @@ define amdgpu_kernel void @test_fold_canonicalize_p1_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_p1_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 1.0
@@ -4025,6 +4265,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_n1_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4035,6 +4277,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_n1_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], -1.0
@@ -4045,6 +4289,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f64(ptr addrspace(1) %out)
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_n1_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4055,6 +4301,8 @@ define amdgpu_kernel void @test_fold_canonicalize_n1_f64(ptr addrspace(1) %out)
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_n1_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], -1.0
@@ -4137,6 +4385,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f64(ptr addrspace(1) %
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_literal_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4147,6 +4397,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f64(ptr addrspace(1) %
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_literal_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0x4030000000000000
@@ -4157,6 +4409,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f64(ptr addrspace(1) %
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_literal_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4167,6 +4421,8 @@ define amdgpu_kernel void @test_fold_canonicalize_literal_f64(ptr addrspace(1) %
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_literal_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0x4030000000000000
@@ -4267,6 +4523,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f64(ptr
;
; GFX1251-SDAG-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4277,6 +4535,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f64(ptr
;
; GFX1251-GISEL-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0
@@ -4287,6 +4547,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f64(ptr
;
; GFX1251-SDAG-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4297,6 +4559,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal0_f64(ptr
;
; GFX1251-GISEL-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0
@@ -4382,6 +4646,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f64(ptr ad
;
; GFX1251-SDAG-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4392,6 +4658,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f64(ptr ad
;
; GFX1251-GISEL-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0xfffffffffffff
@@ -4402,6 +4670,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f64(ptr ad
;
; GFX1251-SDAG-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4412,6 +4682,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal0_f64(ptr ad
;
; GFX1251-GISEL-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal0_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0xfffffffffffff
@@ -4498,6 +4770,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f64(ptr
;
; GFX1251-SDAG-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4508,6 +4782,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f64(ptr
;
; GFX1251-GISEL-TRUE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0x8000000000000000
@@ -4518,6 +4794,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f64(ptr
;
; GFX1251-SDAG-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4528,6 +4806,8 @@ define amdgpu_kernel void @test_no_denormals_fold_canonicalize_denormal1_f64(ptr
;
; GFX1251-GISEL-FAKE16-LABEL: test_no_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0x8000000000000000
@@ -4613,6 +4893,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f64(ptr ad
;
; GFX1251-SDAG-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4623,6 +4905,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f64(ptr ad
;
; GFX1251-GISEL-TRUE16-LABEL: test_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0x800fffffffffffff
@@ -4633,6 +4917,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f64(ptr ad
;
; GFX1251-SDAG-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4643,6 +4929,8 @@ define amdgpu_kernel void @test_denormals_fold_canonicalize_denormal1_f64(ptr ad
;
; GFX1251-GISEL-FAKE16-LABEL: test_denormals_fold_canonicalize_denormal1_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0x800fffffffffffff
@@ -4725,6 +5013,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f64(ptr addrspace(1) %out
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_qnan_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4735,6 +5025,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f64(ptr addrspace(1) %out
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_qnan_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0x7ff8000000000000
@@ -4745,6 +5037,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f64(ptr addrspace(1) %out
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_qnan_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4755,6 +5049,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_f64(ptr addrspace(1) %out
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_qnan_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0x7ff8000000000000
@@ -4851,6 +5147,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f64(ptr addrsp
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4861,6 +5159,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f64(ptr addrsp
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], -1
@@ -4871,6 +5171,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f64(ptr addrsp
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4881,6 +5183,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg1_f64(ptr addrsp
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg1_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], -1
@@ -4977,6 +5281,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f64(ptr addrsp
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -4987,6 +5293,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f64(ptr addrsp
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], -2
@@ -4997,6 +5305,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f64(ptr addrsp
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5007,6 +5317,8 @@ define amdgpu_kernel void @test_fold_canonicalize_qnan_value_neg2_f64(ptr addrsp
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_qnan_value_neg2_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], -2
@@ -5124,6 +5436,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f64(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan0_value_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5134,6 +5448,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f64(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan0_value_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0x7ff0000000000001
@@ -5146,6 +5462,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f64(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan0_value_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5156,6 +5474,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan0_value_f64(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan0_value_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0x7ff0000000000001
@@ -5256,6 +5576,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f64(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan1_value_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5266,6 +5588,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f64(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan1_value_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0x7fffffffffffffff
@@ -5276,6 +5600,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f64(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan1_value_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5286,6 +5612,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan1_value_f64(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan1_value_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0x7fffffffffffffff
@@ -5403,6 +5731,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f64(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan2_value_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5413,6 +5743,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f64(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan2_value_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], 0xfff0000000000001
@@ -5425,6 +5757,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f64(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan2_value_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5435,6 +5769,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan2_value_f64(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan2_value_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], 0xfff0000000000001
@@ -5533,6 +5869,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f64(ptr addrspace(
;
; GFX1251-SDAG-TRUE16-LABEL: test_fold_canonicalize_snan3_value_f64:
; GFX1251-SDAG-TRUE16: ; %bb.0:
+; GFX1251-SDAG-TRUE16-NEXT: global_wb
+; GFX1251-SDAG-TRUE16-NEXT: v_nop
; GFX1251-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5543,6 +5881,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f64(ptr addrspace(
;
; GFX1251-GISEL-TRUE16-LABEL: test_fold_canonicalize_snan3_value_f64:
; GFX1251-GISEL-TRUE16: ; %bb.0:
+; GFX1251-GISEL-TRUE16-NEXT: global_wb
+; GFX1251-GISEL-TRUE16-NEXT: v_nop
; GFX1251-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[0:1], -1
@@ -5553,6 +5893,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f64(ptr addrspace(
;
; GFX1251-SDAG-FAKE16-LABEL: test_fold_canonicalize_snan3_value_f64:
; GFX1251-SDAG-FAKE16: ; %bb.0:
+; GFX1251-SDAG-FAKE16-NEXT: global_wb
+; GFX1251-SDAG-FAKE16-NEXT: v_nop
; GFX1251-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, 0
@@ -5563,6 +5905,8 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f64(ptr addrspace(
;
; GFX1251-GISEL-FAKE16-LABEL: test_fold_canonicalize_snan3_value_f64:
; GFX1251-GISEL-FAKE16: ; %bb.0:
+; GFX1251-GISEL-FAKE16-NEXT: global_wb
+; GFX1251-GISEL-FAKE16-NEXT: v_nop
; GFX1251-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[0:1], -1
@@ -5699,6 +6043,8 @@ define amdgpu_kernel void @test_canonicalize_value_f64_flush(ptr addrspace(1) %a
;
; GFX1251-LABEL: test_canonicalize_value_f64_flush:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -5841,6 +6187,8 @@ define amdgpu_kernel void @test_canonicalize_value_f32_flush(ptr addrspace(1) %a
;
; GFX1251-LABEL: test_canonicalize_value_f32_flush:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -6035,6 +6383,8 @@ define amdgpu_kernel void @test_canonicalize_value_f16_flush(ptr addrspace(1) %a
;
; GFX1251-TRUE16-LABEL: test_canonicalize_value_f16_flush:
; GFX1251-TRUE16: ; %bb.0:
+; GFX1251-TRUE16-NEXT: global_wb
+; GFX1251-TRUE16-NEXT: v_nop
; GFX1251-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-TRUE16-NEXT: v_and_b32_e32 v1, 0x3ff, v0
@@ -6047,6 +6397,8 @@ define amdgpu_kernel void @test_canonicalize_value_f16_flush(ptr addrspace(1) %a
;
; GFX1251-FAKE16-LABEL: test_canonicalize_value_f16_flush:
; GFX1251-FAKE16: ; %bb.0:
+; GFX1251-FAKE16-NEXT: global_wb
+; GFX1251-FAKE16-NEXT: v_nop
; GFX1251-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -6205,6 +6557,8 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_flush(ptr addrspace(1)
;
; GFX1251-LABEL: test_canonicalize_value_v2f16_flush:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -6347,6 +6701,8 @@ define amdgpu_kernel void @test_canonicalize_value_f64_denorm(ptr addrspace(1) %
;
; GFX1251-LABEL: test_canonicalize_value_f64_denorm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -6489,6 +6845,8 @@ define amdgpu_kernel void @test_canonicalize_value_f32_denorm(ptr addrspace(1) %
;
; GFX1251-LABEL: test_canonicalize_value_f32_denorm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -6684,6 +7042,8 @@ define amdgpu_kernel void @test_canonicalize_value_f16_denorm(ptr addrspace(1) %
;
; GFX1251-TRUE16-LABEL: test_canonicalize_value_f16_denorm:
; GFX1251-TRUE16: ; %bb.0:
+; GFX1251-TRUE16-NEXT: global_wb
+; GFX1251-TRUE16-NEXT: v_nop
; GFX1251-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-TRUE16-NEXT: v_and_b32_e32 v1, 0x3ff, v0
@@ -6696,6 +7056,8 @@ define amdgpu_kernel void @test_canonicalize_value_f16_denorm(ptr addrspace(1) %
;
; GFX1251-FAKE16-LABEL: test_canonicalize_value_f16_denorm:
; GFX1251-FAKE16: ; %bb.0:
+; GFX1251-FAKE16-NEXT: global_wb
+; GFX1251-FAKE16-NEXT: v_nop
; GFX1251-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -6853,6 +7215,8 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_denorm(ptr addrspace(1)
;
; GFX1251-LABEL: test_canonicalize_value_v2f16_denorm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -7038,6 +7402,8 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f64(ptr addrspace(1) %out)
;
; GFX1251-LABEL: v_test_canonicalize_var_v2f64:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll b/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll
index 19d2f865fb3ac..c2e915eef7f0b 100644
--- a/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll
+++ b/llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll
@@ -6,6 +6,8 @@
define amdgpu_kernel void @fence_barrier_latency_test(ptr addrspace(1) %global_ptr, i32 %offset, <8 x i32> %tdesc, <4 x i32> %src_desc) #0 {
; CHECK-LABEL: fence_barrier_latency_test:
; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_clause 0x2
; CHECK-NEXT: s_load_b128 s[0:3], s[4:5], 0x40 nv
diff --git a/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll b/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll
index 82ccb5b4aa7e1..6af48b537be89 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll
@@ -13,6 +13,8 @@
define amdgpu_kernel void @test_move_load_address_to_vgpr(ptr addrspace(1) nocapture %arg1, ptr nocapture %arg2) {
; GCN-LABEL: test_move_load_address_to_vgpr:
; GCN: ; %bb.0: ; %bb
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GCN-NEXT: v_mov_b32_e32 v3, 0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index 188b830209713..6bb9914f7a6a4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -10,6 +10,8 @@
define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -53,6 +55,8 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -97,6 +101,8 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -146,6 +152,8 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase
define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -189,6 +197,8 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_2048:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -233,6 +243,8 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voff
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -290,6 +302,8 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
@@ -306,6 +320,8 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
@@ -361,6 +377,8 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i
define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
@@ -377,6 +395,8 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
@@ -433,6 +453,8 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %
define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
@@ -449,6 +471,8 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
@@ -503,6 +527,8 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset,
define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
@@ -519,6 +545,8 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
@@ -581,6 +609,8 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32
define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -626,6 +656,8 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -762,6 +794,8 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -810,6 +844,8 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -956,6 +992,8 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -993,6 +1031,8 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -1104,6 +1144,8 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -1143,6 +1185,8 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -1269,6 +1313,8 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1312,6 +1358,8 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1362,6 +1410,8 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1404,6 +1454,8 @@ define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1453,6 +1505,8 @@ define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -1498,6 +1552,8 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -1638,6 +1694,8 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -1686,6 +1744,8 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -1836,6 +1896,8 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -1876,6 +1938,8 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -1998,6 +2062,8 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -2040,6 +2106,8 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -2177,6 +2245,8 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2220,6 +2290,8 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2270,6 +2342,8 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2312,6 +2386,8 @@ define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2361,6 +2437,8 @@ define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -2406,6 +2484,8 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -2548,6 +2628,8 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -2596,6 +2678,8 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -2748,6 +2832,8 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -2788,6 +2874,8 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -2912,6 +3000,8 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -2954,6 +3044,8 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -3093,6 +3185,8 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3136,6 +3230,8 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3186,6 +3282,8 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3228,6 +3326,8 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3277,6 +3377,8 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -3323,6 +3425,8 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -3464,6 +3568,8 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -3513,6 +3619,8 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -3664,6 +3772,8 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -3705,6 +3815,8 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -3828,6 +3940,8 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -3871,6 +3985,8 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -4009,6 +4125,8 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -4052,6 +4170,8 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i3
define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -4102,6 +4222,8 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voff
define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -4144,6 +4266,8 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -4193,6 +4317,8 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -4239,6 +4365,8 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -4380,6 +4508,8 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -4429,6 +4559,8 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -4580,6 +4712,8 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -4621,6 +4755,8 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -4744,6 +4880,8 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -4787,6 +4925,8 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -4925,6 +5065,8 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -4968,6 +5110,8 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -5018,6 +5162,8 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -5060,6 +5206,8 @@ define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -5109,6 +5257,8 @@ define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -5155,6 +5305,8 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -5296,6 +5448,8 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -5345,6 +5499,8 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -5496,6 +5652,8 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -5537,6 +5695,8 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -5660,6 +5820,8 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -5703,6 +5865,8 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -5841,6 +6005,8 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -5874,6 +6040,8 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -5914,6 +6082,8 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -5946,6 +6116,8 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -5985,6 +6157,8 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -6026,6 +6200,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -6162,6 +6338,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -6206,6 +6384,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -6352,6 +6532,8 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -6389,6 +6571,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -6506,6 +6690,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -6545,6 +6731,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -6677,6 +6865,8 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -6710,6 +6900,8 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -6750,6 +6942,8 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -6782,6 +6976,8 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -6821,6 +7017,8 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -6862,6 +7060,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -6998,6 +7198,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -7042,6 +7244,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -7188,6 +7392,8 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -7225,6 +7431,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -7342,6 +7550,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -7381,6 +7591,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -7513,6 +7725,8 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -7546,6 +7760,8 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -7586,6 +7802,8 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -7618,6 +7836,8 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -7657,6 +7877,8 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -7698,6 +7920,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -7834,6 +8058,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -7878,6 +8104,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -8024,6 +8252,8 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -8061,6 +8291,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -8178,6 +8410,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -8217,6 +8451,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -8349,6 +8585,8 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -8382,6 +8620,8 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -8422,6 +8662,8 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -8454,6 +8696,8 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
@@ -8493,6 +8737,8 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -8534,6 +8780,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -8670,6 +8918,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -8714,6 +8964,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -8860,6 +9112,8 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -8897,6 +9151,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -9014,6 +9270,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -9053,6 +9311,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -9185,6 +9445,8 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
@@ -9231,6 +9493,8 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffse
define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
@@ -9284,6 +9548,8 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32
define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
@@ -9328,6 +9594,8 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffs
define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
@@ -9379,6 +9647,8 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32
define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v4
@@ -9426,6 +9696,8 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -9577,6 +9849,8 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -9627,6 +9901,8 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -9788,6 +10064,8 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v4
@@ -9830,6 +10108,8 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
@@ -9962,6 +10242,8 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -10006,6 +10288,8 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
@@ -10153,6 +10437,8 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -10186,6 +10472,8 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -10226,6 +10514,8 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
@@ -10255,6 +10545,8 @@ define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
@@ -10291,6 +10583,8 @@ define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -10336,6 +10630,8 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -10482,6 +10778,8 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -10530,6 +10828,8 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -10686,6 +10986,8 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -10724,6 +11026,8 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -10846,6 +11150,8 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -10886,6 +11192,8 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -11024,6 +11332,8 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_rtn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -11057,6 +11367,8 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -11097,6 +11409,8 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_nortn:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
@@ -11126,6 +11440,8 @@ define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
@@ -11162,6 +11478,8 @@ define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -11209,6 +11527,8 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -11359,6 +11679,8 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -11409,6 +11731,8 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
@@ -11569,6 +11893,8 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -11609,6 +11935,8 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -11735,6 +12063,8 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -11777,6 +12107,8 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 53bd449502e16..2c07a87add6ba 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -17,6 +17,8 @@
define amdgpu_ps float @flat_load_saddr_i8_offset_0(ptr inreg %sbase) {
; GFX1250-LABEL: flat_load_saddr_i8_offset_0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
@@ -32,6 +34,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0(ptr inreg %sbase) {
define amdgpu_ps float @flat_load_saddr_i8_offset_8388607(ptr inreg %sbase) {
; GFX1250-LABEL: flat_load_saddr_i8_offset_8388607:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607
@@ -48,6 +52,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_8388607(ptr inreg %sbase) {
define amdgpu_ps float @flat_load_saddr_i8_offset_8388608(ptr inreg %sbase) {
; GFX1250-LABEL: flat_load_saddr_i8_offset_8388608:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
@@ -64,6 +70,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_8388608(ptr inreg %sbase) {
define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388608(ptr inreg %sbase) {
; GFX1250-LABEL: flat_load_saddr_i8_offset_neg8388608:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388608
@@ -80,6 +88,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388608(ptr inreg %sbase) {
define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388609(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg8388609:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0xff800000, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -90,6 +100,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388609(ptr inreg %sbase) {
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg8388609:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0xff7fffff
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -1
@@ -101,6 +113,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388609(ptr inreg %sbase) {
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_neg8388609:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_add_co_u32 v0, s0, 0xff800000, s2
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -118,6 +132,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg8388609(ptr inreg %sbase) {
define amdgpu_ps float @flat_load_saddr_i8_offset_0xFFFFFFFF(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0xFFFFFFFF:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0xff800000, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -128,6 +144,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0xFFFFFFFF(ptr inreg %sbase) {
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0xFFFFFFFF:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, -1
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 0
@@ -139,6 +157,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0xFFFFFFFF(ptr inreg %sbase) {
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_0xFFFFFFFF:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_add_co_u32 v0, s0, 0xff800000, s2
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -156,6 +176,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0xFFFFFFFF(ptr inreg %sbase) {
define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000000(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100000000:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_add_co_i32 s3, s3, 1
@@ -165,6 +187,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000000(ptr inreg %sbase)
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100000000:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1
@@ -176,6 +200,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000000(ptr inreg %sbase)
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_0x100000000:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NOECC-NEXT: s_add_co_i32 s3, s3, 1
@@ -192,6 +218,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000000(ptr inreg %sbase)
define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000001(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100000001:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -202,6 +230,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000001(ptr inreg %sbase)
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100000001:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 1
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1
@@ -213,6 +243,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000001(ptr inreg %sbase)
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_0x100000001:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -230,6 +262,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000001(ptr inreg %sbase)
define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000FFF(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100000FFF:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -240,6 +274,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000FFF(ptr inreg %sbase)
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100000FFF:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0xfff
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1
@@ -251,6 +287,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000FFF(ptr inreg %sbase)
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_0x100000FFF:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -268,6 +306,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100000FFF(ptr inreg %sbase)
define amdgpu_ps float @flat_load_saddr_i8_offset_0x100001000(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_0x100001000:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -278,6 +318,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100001000(ptr inreg %sbase)
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_0x100001000:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0x1000
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 1
@@ -289,6 +331,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100001000(ptr inreg %sbase)
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_0x100001000:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -306,6 +350,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_0x100001000(ptr inreg %sbase)
define amdgpu_ps float @flat_load_saddr_i8_offset_neg0xFFFFFFFF(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg0xFFFFFFFF:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0x800000, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -316,6 +362,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0xFFFFFFFF(ptr inreg %sbase
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg0xFFFFFFFF:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 1
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -1
@@ -327,6 +375,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0xFFFFFFFF(ptr inreg %sbase
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_neg0xFFFFFFFF:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_add_co_u32 v0, s0, 0x800000, s2
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -344,6 +394,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0xFFFFFFFF(ptr inreg %sbase
define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000000(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg0x100000000:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_add_co_i32 s3, s3, -1
@@ -353,6 +405,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000000(ptr inreg %sbas
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg0x100000000:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, 0
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -1
@@ -364,6 +418,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000000(ptr inreg %sbas
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_neg0x100000000:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NOECC-NEXT: s_add_co_i32 s3, s3, -1
@@ -380,6 +436,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000000(ptr inreg %sbas
define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000001(ptr inreg %sbase) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_offset_neg0x100000001:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -390,6 +448,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000001(ptr inreg %sbas
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_offset_neg0x100000001:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, -1
; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, -2
@@ -401,6 +461,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000001(ptr inreg %sbas
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_offset_neg0x100000001:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_add_co_u32 v0, s0, 0, s2
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -423,6 +485,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000001(ptr inreg %sbas
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -439,6 +503,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr(ptr inreg %sbase, i32 %voff
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -456,6 +522,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607(ptr inreg %s
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388608(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388608:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -469,6 +537,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388608(ptr inreg %s
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388608:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -483,6 +553,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388608(ptr inreg %s
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388608:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -506,6 +578,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388608(ptr inreg %s
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388608(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388608:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388608
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -523,6 +597,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388608(ptr inreg
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388607(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388607:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388607
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -539,6 +615,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388607(ptr inreg
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -556,6 +634,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order(pt
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -574,6 +654,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint(ptr inreg %sbase,
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -592,6 +674,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr in
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -611,6 +695,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_of
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -636,6 +722,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_of
define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1
@@ -648,6 +736,8 @@ define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
;
; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1
@@ -661,6 +751,8 @@ define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
;
; GFX1250-NOECC-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NOECC-NEXT: ds_load_b64 v[2:3], v1
@@ -683,6 +775,8 @@ define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1
@@ -695,6 +789,8 @@ define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
;
; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1
@@ -708,6 +804,8 @@ define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
;
; GFX1250-NOECC-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NOECC-NEXT: ds_load_b64 v[2:3], v1
@@ -731,6 +829,8 @@ define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset(ptr inreg %sbase, i32 inreg %soffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_uniform_offset:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s4
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
@@ -748,6 +848,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset(ptr inreg %sbase,
define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset_immoffset(ptr inreg %sbase, i32 inreg %soffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_uniform_offset_immoffset:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s4
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-24
@@ -766,6 +868,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset_immoffset(ptr inr
define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 inreg %soffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s4
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
@@ -785,6 +889,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr in
define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 inreg %soffset) {
; GFX1250-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s4
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128
@@ -805,6 +911,8 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_of
define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffset) {
; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -815,6 +923,8 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffse
;
; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -828,6 +938,8 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffse
;
; GFX1250-NOECC-LABEL: flat_load_i8_vgpr64_sgpr32:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: s_mov_b32 s3, 0
; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -847,6 +959,8 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffse
define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i32 inreg %soffset) {
; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -857,6 +971,8 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i3
;
; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -870,6 +986,8 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i3
;
; GFX1250-NOECC-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: s_mov_b32 s3, 0
; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -894,6 +1012,8 @@ define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i3
define amdgpu_ps float @flat_load_saddr_f32_natural_addressing(ptr inreg %sbase, ptr %voffset.ptr) {
; GFX1250-LABEL: flat_load_saddr_f32_natural_addressing:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -911,6 +1031,8 @@ define amdgpu_ps float @flat_load_saddr_f32_natural_addressing(ptr inreg %sbase,
define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inreg %sbase, ptr %voffset.ptr) {
; GFX1250-LABEL: flat_load_saddr_f32_natural_addressing_immoffset:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -929,6 +1051,8 @@ define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inr
define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, ptr %voffset.ptr) {
; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -946,6 +1070,8 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, pt
define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg %sbase, ptr %voffset.ptr) {
; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range_imm_offset:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -964,6 +1090,8 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg
define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_too_large(ptr inreg %sbase, ptr %voffset.ptr) {
; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range_too_large:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -984,6 +1112,8 @@ define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_too_large(ptr inreg
define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i16:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -991,6 +1121,8 @@ define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -998,6 +1130,8 @@ define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3]
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1005,6 +1139,8 @@ define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16:
; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1019,6 +1155,8 @@ define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i16_immneg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1026,6 +1164,8 @@ define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_immneg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1033,6 +1173,8 @@ define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voff
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_immneg128:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3] offset:-128
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1040,6 +1182,8 @@ define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voff
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16_immneg128:
; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1055,6 +1199,8 @@ define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voff
define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_f16:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1062,6 +1208,8 @@ define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
;
; GFX1250-GISEL-LABEL: flat_load_saddr_f16:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1069,6 +1217,8 @@ define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_f16:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3]
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1076,6 +1226,8 @@ define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_f16:
; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1089,6 +1241,8 @@ define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_f16_immneg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1096,6 +1250,8 @@ define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_load_saddr_f16_immneg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1103,6 +1259,8 @@ define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voff
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_f16_immneg128:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3] offset:-128
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1110,6 +1268,8 @@ define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voff
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_f16_immneg128:
; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1124,6 +1284,8 @@ define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voff
define amdgpu_ps float @flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1138,6 +1300,8 @@ define amdgpu_ps float @flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) {
define amdgpu_ps float @flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1153,6 +1317,8 @@ define amdgpu_ps float @flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps float @flat_load_saddr_f32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1166,6 +1332,8 @@ define amdgpu_ps float @flat_load_saddr_f32(ptr inreg %sbase, i32 %voffset) {
define amdgpu_ps float @flat_load_saddr_f32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_f32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1180,6 +1348,8 @@ define amdgpu_ps float @flat_load_saddr_f32_immneg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x half> @flat_load_saddr_v2i16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1194,6 +1364,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2i16(ptr inreg %sbase, i32 %voffse
define amdgpu_ps <2 x half> @flat_load_saddr_v2i16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2i16_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1209,6 +1381,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2i16_immneg128(ptr inreg %sbase, i
define amdgpu_ps <2 x half> @flat_load_saddr_v2f16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1222,6 +1396,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2f16(ptr inreg %sbase, i32 %voffse
define amdgpu_ps <2 x half> @flat_load_saddr_v2f16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2f16_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1236,6 +1412,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2f16_immneg128(ptr inreg %sbase, i
define amdgpu_ps <2 x half> @flat_load_saddr_p3(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_p3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1251,6 +1429,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_p3(ptr inreg %sbase, i32 %voffset)
define amdgpu_ps <2 x half> @flat_load_saddr_p3_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_p3_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1267,6 +1447,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_p3_immneg128(ptr inreg %sbase, i32
define amdgpu_ps <2 x float> @flat_load_saddr_f64(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1281,6 +1463,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_f64(ptr inreg %sbase, i32 %voffset
define amdgpu_ps <2 x float> @flat_load_saddr_f64_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_f64_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1296,6 +1480,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_f64_immneg128(ptr inreg %sbase, i3
define amdgpu_ps <2 x float> @flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1310,6 +1496,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset
define amdgpu_ps <2 x float> @flat_load_saddr_i64_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i64_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1325,6 +1513,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_i64_immneg128(ptr inreg %sbase, i3
define amdgpu_ps <2 x float> @flat_load_saddr_v2f32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1338,6 +1528,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2f32(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <2 x float> @flat_load_saddr_v2f32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2f32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1352,6 +1544,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2f32_immneg128(ptr inreg %sbase,
define amdgpu_ps <2 x float> @flat_load_saddr_v2i32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1366,6 +1560,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2i32(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <2 x float> @flat_load_saddr_v2i32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2i32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1381,6 +1577,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2i32_immneg128(ptr inreg %sbase,
define amdgpu_ps <2 x float> @flat_load_saddr_v4i16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1395,6 +1593,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v4i16(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <2 x float> @flat_load_saddr_v4i16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4i16_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1410,6 +1610,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v4i16_immneg128(ptr inreg %sbase,
define amdgpu_ps <2 x float> @flat_load_saddr_v4f16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1424,6 +1626,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v4f16(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <2 x float> @flat_load_saddr_v4f16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4f16_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1439,6 +1643,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v4f16_immneg128(ptr inreg %sbase,
define amdgpu_ps <2 x float> @flat_load_saddr_p1(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_p1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1454,6 +1660,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_p1(ptr inreg %sbase, i32 %voffset)
define amdgpu_ps <2 x float> @flat_load_saddr_p1_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_p1_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1470,6 +1678,8 @@ define amdgpu_ps <2 x float> @flat_load_saddr_p1_immneg128(ptr inreg %sbase, i32
define amdgpu_ps <3 x float> @flat_load_saddr_v3f32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v3f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1483,6 +1693,8 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3f32(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <3 x float> @flat_load_saddr_v3f32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v3f32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1497,6 +1709,8 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3f32_immneg128(ptr inreg %sbase,
define amdgpu_ps <3 x float> @flat_load_saddr_v3i32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v3i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1511,6 +1725,8 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3i32(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <3 x float> @flat_load_saddr_v3i32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v3i32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1526,6 +1742,8 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3i32_immneg128(ptr inreg %sbase,
define amdgpu_ps <6 x half> @flat_load_saddr_v6f16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v6f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1539,6 +1757,8 @@ define amdgpu_ps <6 x half> @flat_load_saddr_v6f16(ptr inreg %sbase, i32 %voffse
define amdgpu_ps <6 x half> @flat_load_saddr_v6f16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v6f16_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1553,6 +1773,8 @@ define amdgpu_ps <6 x half> @flat_load_saddr_v6f16_immneg128(ptr inreg %sbase, i
define amdgpu_ps <4 x float> @flat_load_saddr_v4f32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1566,6 +1788,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4f32(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <4 x float> @flat_load_saddr_v4f32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4f32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1580,6 +1804,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4f32_immneg128(ptr inreg %sbase,
define amdgpu_ps <4 x float> @flat_load_saddr_v4i32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1594,6 +1820,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4i32(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <4 x float> @flat_load_saddr_v4i32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4i32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1609,6 +1837,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4i32_immneg128(ptr inreg %sbase,
define amdgpu_ps <4 x float> @flat_load_saddr_v2i64(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1623,6 +1853,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2i64(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <4 x float> @flat_load_saddr_v2i64_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2i64_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1638,6 +1870,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2i64_immneg128(ptr inreg %sbase,
define amdgpu_ps <4 x float> @flat_load_saddr_i128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1652,6 +1886,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_i128(ptr inreg %sbase, i32 %voffse
define amdgpu_ps <4 x float> @flat_load_saddr_i128_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_i128_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1667,6 +1903,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_i128_immneg128(ptr inreg %sbase, i
define amdgpu_ps <4 x float> @flat_load_saddr_v2p1(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2p1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1682,6 +1920,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2p1(ptr inreg %sbase, i32 %voffse
define amdgpu_ps <4 x float> @flat_load_saddr_v2p1_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v2p1_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1698,6 +1938,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2p1_immneg128(ptr inreg %sbase, i
define amdgpu_ps <4 x float> @flat_load_saddr_v4p3(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4p3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1713,6 +1955,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4p3(ptr inreg %sbase, i32 %voffse
define amdgpu_ps <4 x float> @flat_load_saddr_v4p3_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_load_saddr_v4p3_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1733,6 +1977,8 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4p3_immneg128(ptr inreg %sbase, i
define amdgpu_ps float @flat_sextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_sextload_saddr_i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1748,6 +1994,8 @@ define amdgpu_ps float @flat_sextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
define amdgpu_ps float @flat_sextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_sextload_saddr_i8_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1764,6 +2012,8 @@ define amdgpu_ps float @flat_sextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %
define amdgpu_ps float @flat_sextload_saddr_i16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_sextload_saddr_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_i16 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1779,6 +2029,8 @@ define amdgpu_ps float @flat_sextload_saddr_i16(ptr inreg %sbase, i32 %voffset)
define amdgpu_ps float @flat_sextload_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_sextload_saddr_i16_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_i16 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1795,6 +2047,8 @@ define amdgpu_ps float @flat_sextload_saddr_i16_immneg128(ptr inreg %sbase, i32
define amdgpu_ps float @flat_zextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_zextload_saddr_i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1810,6 +2064,8 @@ define amdgpu_ps float @flat_zextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
define amdgpu_ps float @flat_zextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_zextload_saddr_i8_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1826,6 +2082,8 @@ define amdgpu_ps float @flat_zextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %
define amdgpu_ps float @flat_zextload_saddr_i16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_zextload_saddr_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1841,6 +2099,8 @@ define amdgpu_ps float @flat_zextload_saddr_i16(ptr inreg %sbase, i32 %voffset)
define amdgpu_ps float @flat_zextload_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: flat_zextload_saddr_i16_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1861,6 +2121,8 @@ define amdgpu_ps float @flat_zextload_saddr_i16_immneg128(ptr inreg %sbase, i32
define amdgpu_ps float @atomic_flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: atomic_flat_load_saddr_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1877,6 +2139,8 @@ define amdgpu_ps float @atomic_flat_load_saddr_i32(ptr inreg %sbase, i32 %voffse
define amdgpu_ps float @atomic_flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: atomic_flat_load_saddr_i32_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1894,6 +2158,8 @@ define amdgpu_ps float @atomic_flat_load_saddr_i32_immneg128(ptr inreg %sbase, i
define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: atomic_flat_load_saddr_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1910,6 +2176,8 @@ define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64(ptr inreg %sbase, i32 %
define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-LABEL: atomic_flat_load_saddr_i64_immneg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1931,6 +2199,8 @@ define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64_immneg128(ptr inreg %sb
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_undef_hi:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1938,6 +2208,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_undef_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1948,6 +2220,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_undef_hi:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1955,6 +2229,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_undef_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_b16 v0, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1970,6 +2246,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-LABEL: flat_load_saddr_i16_d16lo_undef_hi_immneg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1977,6 +2255,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_undef_hi_immneg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1987,6 +2267,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_undef_hi_immneg128:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1994,6 +2276,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_undef_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_b16 v0, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2010,6 +2294,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2018,6 +2304,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2026,6 +2314,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2034,6 +2324,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2042,6 +2334,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_zero_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3]
@@ -2059,6 +2353,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2067,6 +2363,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi_immneg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2075,6 +2373,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2083,6 +2383,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zero_hi_immneg128:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2091,6 +2393,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_zero_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3] offset:-128
@@ -2109,6 +2413,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2117,6 +2423,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2125,6 +2433,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2133,6 +2443,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2149,6 +2461,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2157,6 +2471,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2165,6 +2481,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2173,6 +2491,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2190,6 +2510,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2198,6 +2520,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2206,6 +2530,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2214,6 +2540,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_u8 v1, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2231,6 +2559,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2239,6 +2569,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2247,6 +2579,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2255,6 +2589,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_u8 v1, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2273,6 +2609,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2281,6 +2619,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2291,6 +2631,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2299,6 +2641,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2309,6 +2653,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_i8 v1, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2326,6 +2672,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2334,6 +2682,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2344,6 +2694,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2352,6 +2704,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2362,6 +2716,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_i8 v1, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2384,6 +2740,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2392,6 +2750,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e64 v1, 0xffff, s0
@@ -2402,6 +2762,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2410,6 +2772,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2418,6 +2782,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_undef_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v0, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2433,6 +2799,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2441,6 +2809,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi_immneg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e64 v1, 0xffff, s0
@@ -2451,6 +2821,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2459,6 +2831,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_undef_hi_immneg128:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2467,6 +2841,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_undef_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v0, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2483,6 +2859,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2491,6 +2869,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2499,6 +2879,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
@@ -2509,6 +2891,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3]
@@ -2518,6 +2902,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3]
@@ -2535,6 +2921,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inreg %sbase, i32 %voffset) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2543,6 +2931,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2551,6 +2941,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
@@ -2561,6 +2953,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3] offset:-128
@@ -2570,6 +2964,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3] offset:-128
@@ -2588,6 +2984,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2596,6 +2994,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2606,6 +3006,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2616,6 +3018,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2632,6 +3036,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2640,6 +3046,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2650,6 +3058,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2660,6 +3070,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2677,6 +3089,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2685,6 +3099,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2695,6 +3111,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2705,6 +3123,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_u8 v1, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2722,6 +3142,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2730,6 +3152,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2740,6 +3164,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2750,6 +3176,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_u8 v1, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2768,6 +3196,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2776,6 +3206,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2787,6 +3219,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2797,6 +3231,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2808,6 +3244,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_i8 v1, v0, s[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2825,6 +3263,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %reg) {
; GFX1250-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2833,6 +3273,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2844,6 +3286,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2854,6 +3298,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2865,6 +3311,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: flat_load_d16_hi_i8 v1, v0, s[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2888,6 +3336,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
define amdgpu_ps float @flat_load_saddr_i8_offset_or_i64_imm_offset_16(ptr addrspace(6) inreg %sbase, i32 %idx) {
; GFX1250-LABEL: flat_load_saddr_i8_offset_or_i64_imm_offset_16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_bitop2_b32 v0, 16, v0 bitop3:0x54
; GFX1250-NEXT: flat_load_u8 v0, v[0:1]
@@ -2905,6 +3355,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_or_i64_imm_offset_16(ptr addrs
define amdgpu_ps float @flat_load_saddr_i8_offset_or_i64_imm_offset_4160(ptr addrspace(6) inreg %sbase, i32 %idx) {
; GFX1250-LABEL: flat_load_saddr_i8_offset_or_i64_imm_offset_4160:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_or_b32_e32 v0, 0x1040, v0
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -2927,6 +3379,8 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_or_i64_imm_offset_4160(ptr add
define amdgpu_ps void @flat_addr_64bit_lsr_iv(ptr inreg %arg) {
; GFX1250-SDAG-LABEL: flat_addr_64bit_lsr_iv:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_movk_i32 s0, 0x100
@@ -2944,6 +3398,8 @@ define amdgpu_ps void @flat_addr_64bit_lsr_iv(ptr inreg %arg) {
;
; GFX1250-GISEL-LABEL: flat_addr_64bit_lsr_iv:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-GISEL-NEXT: s_movk_i32 s0, 0x100
@@ -2962,6 +3418,8 @@ define amdgpu_ps void @flat_addr_64bit_lsr_iv(ptr inreg %arg) {
;
; GFX1250-NOECC-LABEL: flat_addr_64bit_lsr_iv:
; GFX1250-NOECC: ; %bb.0: ; %bb
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NOECC-NEXT: s_movk_i32 s0, 0x100
@@ -2997,6 +3455,8 @@ bb3: ; preds = %bb3, %bb
define amdgpu_ps void @flat_addr_64bit_lsr_iv_multiload(ptr inreg %arg, ptr inreg %arg.1) {
; GFX1250-SDAG-LABEL: flat_addr_64bit_lsr_iv_multiload:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_movk_i32 s0, 0x100
@@ -3016,6 +3476,8 @@ define amdgpu_ps void @flat_addr_64bit_lsr_iv_multiload(ptr inreg %arg, ptr inre
;
; GFX1250-GISEL-LABEL: flat_addr_64bit_lsr_iv_multiload:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-GISEL-NEXT: s_movk_i32 s0, 0x100
@@ -3036,6 +3498,8 @@ define amdgpu_ps void @flat_addr_64bit_lsr_iv_multiload(ptr inreg %arg, ptr inre
;
; GFX1250-NOECC-LABEL: flat_addr_64bit_lsr_iv_multiload:
; GFX1250-NOECC: ; %bb.0: ; %bb
+; GFX1250-NOECC-NEXT: global_wb
+; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NOECC-NEXT: s_movk_i32 s0, 0x100
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
index 28efba68b9a93..4f001ed14849d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
@@ -9,6 +9,8 @@
define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {
; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -25,6 +27,8 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voff
define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {
; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_2047:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -42,6 +46,8 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbas
define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {
; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_neg2048:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -65,6 +71,8 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
@@ -76,6 +84,8 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %d
;
; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
@@ -96,6 +106,8 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %d
define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset, i8 %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
@@ -107,6 +119,8 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
;
; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
@@ -131,6 +145,8 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, i16 %data) {
; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
@@ -143,6 +159,8 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i16 %data) {
; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
@@ -156,6 +174,8 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, half %data) {
; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
@@ -168,6 +188,8 @@ define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, half %data) {
; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
@@ -181,6 +203,8 @@ define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
@@ -193,6 +217,8 @@ define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr_offset_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
@@ -206,6 +232,8 @@ define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, float %data) {
; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
@@ -218,6 +246,8 @@ define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, float %data) {
; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr_offset_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
@@ -231,6 +261,8 @@ define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr_offset_neg128(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) {
; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
@@ -243,6 +275,8 @@ define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr(ptr inreg %sbase, i32 %voff
define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) {
; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr_offset_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
@@ -256,6 +290,8 @@ define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr_offset_neg128(ptr inreg %sb
define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_i64_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
@@ -263,6 +299,8 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
@@ -276,6 +314,8 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_i64_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
@@ -283,6 +323,8 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %s
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
@@ -297,6 +339,8 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr(ptr inreg %sbase, i32 %voffset, double %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_f64_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
@@ -304,6 +348,8 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
@@ -317,6 +363,8 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, double %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_f64_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
@@ -324,6 +372,8 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr_offset_neg128(ptr inreg %s
;
; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
@@ -338,6 +388,8 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr_offset_neg128(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i32> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2i32_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
@@ -345,6 +397,8 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
@@ -358,6 +412,8 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i32> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2i32_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
@@ -365,6 +421,8 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
@@ -379,6 +437,8 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x float> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2f32_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
@@ -386,6 +446,8 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
@@ -399,6 +461,8 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x float> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2f32_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
@@ -406,6 +470,8 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
@@ -420,6 +486,8 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4i16_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
@@ -427,6 +495,8 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
@@ -440,6 +510,8 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4i16_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
@@ -447,6 +519,8 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
@@ -461,6 +535,8 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x half> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4f16_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
@@ -468,6 +544,8 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
@@ -481,6 +559,8 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x half> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4f16_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
@@ -488,6 +568,8 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
@@ -502,6 +584,8 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr(ptr inreg %sbase, i32 %voffset, ptr %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_p1_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
@@ -509,6 +593,8 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
@@ -522,6 +608,8 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr(ptr inreg %sbase, i32 %voff
define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, ptr %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_p1_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
@@ -529,6 +617,8 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr_offset_neg128(ptr inreg %sb
;
; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
@@ -543,6 +633,8 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr_offset_neg128(ptr inreg %sb
define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <3 x i32> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v3i32_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -551,6 +643,8 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -565,6 +659,8 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <3 x i32> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v3i32_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -573,6 +669,8 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -588,6 +686,8 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <3 x float> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v3f32_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -596,6 +696,8 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -610,6 +712,8 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <3 x float> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v3f32_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -618,6 +722,8 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -633,6 +739,8 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <6 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v6i16_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -641,6 +749,8 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -655,6 +765,8 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <6 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v6i16_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -663,6 +775,8 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -678,6 +792,8 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <6 x half> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v6f16_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -686,6 +802,8 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -700,6 +818,8 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <6 x half> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v6f16_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
@@ -708,6 +828,8 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
@@ -723,6 +845,8 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x i32> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4i32_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -731,6 +855,8 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -745,6 +871,8 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x i32> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4i32_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -753,6 +881,8 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -768,6 +898,8 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x float> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4f32_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -776,6 +908,8 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -790,6 +924,8 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x float> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4f32_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -798,6 +934,8 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -813,6 +951,8 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i64> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2i64_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -821,6 +961,8 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -835,6 +977,8 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i64> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2i64_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -843,6 +987,8 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -858,6 +1004,8 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x double> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2f64_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -866,6 +1014,8 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -880,6 +1030,8 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x double> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2f64_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -888,6 +1040,8 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -903,6 +1057,8 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <8 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v8i16_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -911,6 +1067,8 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -925,6 +1083,8 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <8 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v8i16_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -933,6 +1093,8 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -948,6 +1110,8 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <8 x half> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v8f16_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -956,6 +1120,8 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -970,6 +1136,8 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr(ptr inreg %sbase, i32 %v
define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <8 x half> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v8f16_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -978,6 +1146,8 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr_offset_neg128(ptr inreg
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -993,6 +1163,8 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr_offset_neg128(ptr inreg
define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x ptr> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2p1_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -1001,6 +1173,8 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -1015,6 +1189,8 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr(ptr inreg %sbase, i32 %vo
define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x ptr> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v2p1_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -1023,6 +1199,8 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr_offset_neg128(ptr inreg %
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -1038,6 +1216,8 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr_offset_neg128(ptr inreg %
define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x ptr addrspace(3)> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4p3_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -1046,6 +1226,8 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -1060,6 +1242,8 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr(ptr inreg %sbase, i32 %vo
define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x ptr addrspace(3)> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_v4p3_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -1068,6 +1252,8 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
@@ -1087,6 +1273,8 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %
define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1101,6 +1289,8 @@ define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i
define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1116,6 +1306,8 @@ define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr i
define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i64_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
@@ -1125,6 +1317,8 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i
;
; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
@@ -1140,6 +1334,8 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i
define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
@@ -1149,6 +1345,8 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr i
;
; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
@@ -1169,18 +1367,24 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr i
define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-REAL16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v0, v1, s[2:3]
@@ -1195,18 +1399,24 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
; GFX1250-SDAG-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-REAL16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128
@@ -1222,6 +1432,8 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3]
; GFX1250-NEXT: s_endpgm
@@ -1236,6 +1448,8 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr(ptr inreg %s
define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3.ll b/llvm/test/CodeGen/AMDGPU/fmax3.ll
index 7d7bd9e06f8e4..e539a608d0618 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3.ll
@@ -194,6 +194,8 @@ define amdgpu_kernel void @test_fmax3_olt_0_f32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: test_fmax3_olt_0_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, -1
@@ -415,6 +417,8 @@ define amdgpu_kernel void @test_fmax3_olt_1_f32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: test_fmax3_olt_1_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, -1
@@ -734,6 +738,8 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-TRUE16-LABEL: test_fmax3_olt_0_f16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -765,6 +771,8 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-FAKE16-LABEL: test_fmax3_olt_0_f16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: s_mov_b32 s2, -1
@@ -1085,6 +1093,8 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-TRUE16-LABEL: test_fmax3_olt_1_f16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -1116,6 +1126,8 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-FAKE16-LABEL: test_fmax3_olt_1_f16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: s_mov_b32 s2, -1
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3.ll b/llvm/test/CodeGen/AMDGPU/fmin3.ll
index 8abb1b4178212..c0789bb0ef7e2 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3.ll
@@ -194,6 +194,8 @@ define amdgpu_kernel void @test_fmin3_olt_0_f32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: test_fmin3_olt_0_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, -1
@@ -415,6 +417,8 @@ define amdgpu_kernel void @test_fmin3_olt_1_f32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: test_fmin3_olt_1_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, -1
@@ -734,6 +738,8 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-TRUE16-LABEL: test_fmin3_olt_0_f16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -765,6 +771,8 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-FAKE16-LABEL: test_fmin3_olt_0_f16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: s_mov_b32 s2, -1
@@ -1085,6 +1093,8 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-TRUE16-LABEL: test_fmin3_olt_1_f16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-TRUE16-NEXT: s_mov_b32 s2, -1
@@ -1116,6 +1126,8 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-FAKE16-LABEL: test_fmin3_olt_1_f16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: s_mov_b32 s2, -1
@@ -1459,6 +1471,8 @@ define amdgpu_kernel void @test_fmin3_olt_0_f64(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: test_fmin3_olt_0_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, -1
@@ -1702,6 +1716,8 @@ define amdgpu_kernel void @test_fmin3_olt_1_f64(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: test_fmin3_olt_1_f64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s2, -1
diff --git a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx942.ll b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx942.ll
index ebcf9b7e2a1d6..4f9712ab338f0 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-atomics-gfx942.ll
@@ -35,6 +35,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat(ptr %ptr) {
;
; GFX1250-LABEL: flat_atomic_fadd_f32_noret_pat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4.0
@@ -79,6 +81,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat_ieee(ptr %ptr) #0 {
;
; GFX1250-LABEL: flat_atomic_fadd_f32_noret_pat_ieee:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4.0
@@ -160,6 +164,8 @@ define amdgpu_kernel void @local_atomic_fadd_v2f16_noret(ptr addrspace(3) %ptr,
;
; GFX1250-LABEL: local_atomic_fadd_v2f16_noret:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -227,6 +233,8 @@ define amdgpu_kernel void @local_atomic_fadd_v2bf16_noret(ptr addrspace(3) %ptr,
;
; GFX1250-LABEL: local_atomic_fadd_v2bf16_noret:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index 8f03abd1124f7..a4727d8b1eb51 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -42,6 +42,8 @@ define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -73,6 +75,8 @@ define amdgpu_ps void @raw_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -117,6 +121,8 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -164,6 +170,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -195,6 +203,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -239,6 +249,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -286,6 +298,8 @@ define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -317,6 +331,8 @@ define amdgpu_ps void @struct_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -361,6 +377,8 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -407,6 +425,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -438,6 +458,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -482,6 +504,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -528,6 +552,8 @@ define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -559,6 +585,8 @@ define amdgpu_ps void @raw_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -603,6 +631,8 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -650,6 +680,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -681,6 +713,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -725,6 +759,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -772,6 +808,8 @@ define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -803,6 +841,8 @@ define amdgpu_ps void @struct_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -847,6 +887,8 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -893,6 +935,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -924,6 +968,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -968,6 +1014,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1014,6 +1062,8 @@ define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1045,6 +1095,8 @@ define amdgpu_ps void @raw_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1089,6 +1141,8 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1136,6 +1190,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1167,6 +1223,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1211,6 +1269,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1258,6 +1318,8 @@ define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1289,6 +1351,8 @@ define amdgpu_ps void @struct_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1333,6 +1397,8 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1379,6 +1445,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1410,6 +1478,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1454,6 +1524,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1504,6 +1576,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -1548,6 +1622,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -1594,6 +1670,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -1638,6 +1716,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -1819,6 +1899,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -1867,6 +1949,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -1912,6 +1996,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent(ptr %ptr) #1 {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -1960,6 +2046,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_system:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2144,6 +2232,8 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
;
; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2186,6 +2276,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret(ptr addrspace(3) %ptr, do
;
; GFX1250-LABEL: local_atomic_fadd_f64_noret:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
@@ -2258,6 +2350,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
;
; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2295,6 +2389,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
;
; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
@@ -2332,6 +2428,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
;
; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.0
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index f57fa36b98bbe..4a0fd748bb1cf 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -199,6 +199,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -219,6 +221,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -239,6 +243,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -256,6 +262,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -462,6 +470,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -482,6 +492,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -502,6 +514,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -519,6 +533,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -1242,6 +1258,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -1310,6 +1328,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -1378,6 +1398,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -1437,6 +1459,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -1701,6 +1725,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -1723,6 +1749,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -1745,6 +1773,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -1763,6 +1793,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -2016,6 +2048,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -2035,6 +2069,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -2054,6 +2090,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -2069,6 +2107,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -3325,6 +3365,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -3442,6 +3484,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -3559,6 +3603,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -3665,6 +3711,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -4039,6 +4087,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -4061,6 +4111,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -4083,6 +4135,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -4106,6 +4160,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -4318,6 +4374,8 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -4340,6 +4398,8 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -4362,6 +4422,8 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -4380,6 +4442,8 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -4588,6 +4652,8 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -4610,6 +4676,8 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -4632,6 +4700,8 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -4650,6 +4720,8 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -4858,6 +4930,8 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -4880,6 +4954,8 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -4902,6 +4978,8 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -4920,6 +4998,8 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -5157,6 +5237,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -5179,6 +5261,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -5201,6 +5285,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -5219,6 +5305,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -5456,6 +5544,8 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -5479,6 +5569,8 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -5502,6 +5594,8 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -5521,6 +5615,8 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -5764,6 +5860,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-TRUE16-NEXT: global_wb
+; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -5786,6 +5884,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -5808,6 +5908,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -5826,6 +5928,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/global-address.ll b/llvm/test/CodeGen/AMDGPU/global-address.ll
index 1a232b6c3fec3..79f2a607aef60 100644
--- a/llvm/test/CodeGen/AMDGPU/global-address.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-address.ll
@@ -27,6 +27,8 @@ define amdgpu_kernel void @caller_internal() {
;
; GFX1250-PAL-LABEL: caller_internal:
; GFX1250-PAL: ; %bb.0:
+; GFX1250-PAL-NEXT: global_wb
+; GFX1250-PAL-NEXT: v_nop
; GFX1250-PAL-NEXT: s_mov_b32 s32, 0
; GFX1250-PAL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-PAL-NEXT: s_mov_b64 s[0:1], internal_func at abs64
@@ -47,6 +49,8 @@ define amdgpu_kernel void @caller_internal() {
;
; GFX1250-HSA-LABEL: caller_internal:
; GFX1250-HSA: ; %bb.0:
+; GFX1250-HSA-NEXT: global_wb
+; GFX1250-HSA-NEXT: v_nop
; GFX1250-HSA-NEXT: s_mov_b32 s32, 0
; GFX1250-HSA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-HSA-NEXT: s_get_pc_i64 s[0:1]
@@ -94,6 +98,8 @@ define amdgpu_kernel void @caller_exterinal() {
;
; GFX1250-PAL-LABEL: caller_exterinal:
; GFX1250-PAL: ; %bb.0:
+; GFX1250-PAL-NEXT: global_wb
+; GFX1250-PAL-NEXT: v_nop
; GFX1250-PAL-NEXT: s_mov_b32 s32, 0
; GFX1250-PAL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-PAL-NEXT: v_mov_b32_e32 v31, v0
@@ -124,6 +130,8 @@ define amdgpu_kernel void @caller_exterinal() {
;
; GFX1250-HSA-LABEL: caller_exterinal:
; GFX1250-HSA: ; %bb.0:
+; GFX1250-HSA-NEXT: global_wb
+; GFX1250-HSA-NEXT: v_nop
; GFX1250-HSA-NEXT: s_mov_b32 s32, 0
; GFX1250-HSA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-HSA-NEXT: v_mov_b32_e32 v31, v0
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index fe38200cbf234..91905a777cb6f 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -474,6 +474,8 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2, ptr addrspace(1) %out) {
; GCN-SDAG-LABEL: test_v7i16_load_store_kernel:
; GCN-SDAG: ; %bb.0:
+; GCN-SDAG-NEXT: global_wb
+; GCN-SDAG-NEXT: v_nop
; GCN-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GCN-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -502,6 +504,8 @@ define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1,
;
; GCN-GISEL-LABEL: test_v7i16_load_store_kernel:
; GCN-GISEL: ; %bb.0:
+; GCN-GISEL-NEXT: global_wb
+; GCN-GISEL-NEXT: v_nop
; GCN-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GCN-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-attr.ll b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-attr.ll
index 2f26e036867df..a197ca7b606d4 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-delay-alu-attr.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-delay-alu-attr.ll
@@ -4,6 +4,8 @@
define amdgpu_kernel void @delay_alu() {
; CHECK-LABEL: delay_alu:
; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -22,6 +24,8 @@ entry:
define amdgpu_kernel void @delay_alu_waves_1_8() "amdgpu-waves-per-eu"="1,8" {
; CHECK-LABEL: delay_alu_waves_1_8:
; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -40,6 +44,8 @@ entry:
define amdgpu_kernel void @delay_alu_waves_1_1() "amdgpu-waves-per-eu"="1,1" {
; CHECK-LABEL: delay_alu_waves_1_1:
; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -58,6 +64,8 @@ entry:
define amdgpu_kernel void @delay_alu_waves_1_1_wgsize() "amdgpu-flat-work-group-size"="1,128" "amdgpu-waves-per-eu"="1,1" {
; CHECK-LABEL: delay_alu_waves_1_1_wgsize:
; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT: v_add_nc_u32_e32 v0, 0xff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
index 37d7c24cb6869..9ae7f441d7128 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -66,6 +66,8 @@ define amdgpu_kernel void @s_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr a
;
; GFX1250-LABEL: s_insertelement_v2bf16_0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -141,6 +143,8 @@ define amdgpu_kernel void @s_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr a
;
; GFX1250-LABEL: s_insertelement_v2bf16_1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -225,6 +229,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr a
;
; GFX1250-FAKE16-LABEL: v_insertelement_v2bf16_0:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -239,6 +245,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_0(ptr addrspace(1) %out, ptr a
;
; GFX1250-REAL16-LABEL: v_insertelement_v2bf16_0:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -324,6 +332,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_0_inlineimm(ptr addrspace(1) %
;
; GFX1250-FAKE16-LABEL: v_insertelement_v2bf16_0_inlineimm:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -336,6 +346,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_0_inlineimm(ptr addrspace(1) %
;
; GFX1250-REAL16-LABEL: v_insertelement_v2bf16_0_inlineimm:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -423,6 +435,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr a
;
; GFX1250-FAKE16-LABEL: v_insertelement_v2bf16_1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -437,6 +451,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_1(ptr addrspace(1) %out, ptr a
;
; GFX1250-REAL16-LABEL: v_insertelement_v2bf16_1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -522,6 +538,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_1_inlineimm(ptr addrspace(1) %
;
; GFX1250-FAKE16-LABEL: v_insertelement_v2bf16_1_inlineimm:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -534,6 +552,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_1_inlineimm(ptr addrspace(1) %
;
; GFX1250-REAL16-LABEL: v_insertelement_v2bf16_1_inlineimm:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -646,6 +666,8 @@ define amdgpu_kernel void @v_insertelement_v2bf16_dynamic_vgpr(ptr addrspace(1)
;
; GFX1250-LABEL: v_insertelement_v2bf16_dynamic_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
@@ -749,6 +771,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_0(ptr addrspace(1) %out, ptr a
;
; GFX1250-FAKE16-LABEL: v_insertelement_v4bf16_0:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -763,6 +787,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_0(ptr addrspace(1) %out, ptr a
;
; GFX1250-REAL16-LABEL: v_insertelement_v4bf16_0:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -858,6 +884,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_1(ptr addrspace(1) %out, ptr a
;
; GFX1250-FAKE16-LABEL: v_insertelement_v4bf16_1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -872,6 +900,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_1(ptr addrspace(1) %out, ptr a
;
; GFX1250-REAL16-LABEL: v_insertelement_v4bf16_1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -969,6 +999,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_2(ptr addrspace(1) %out, ptr a
;
; GFX1250-FAKE16-LABEL: v_insertelement_v4bf16_2:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -983,6 +1015,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_2(ptr addrspace(1) %out, ptr a
;
; GFX1250-REAL16-LABEL: v_insertelement_v4bf16_2:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1078,6 +1112,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_3(ptr addrspace(1) %out, ptr a
;
; GFX1250-FAKE16-LABEL: v_insertelement_v4bf16_3:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1092,6 +1128,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_3(ptr addrspace(1) %out, ptr a
;
; GFX1250-REAL16-LABEL: v_insertelement_v4bf16_3:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1210,6 +1248,8 @@ define amdgpu_kernel void @v_insertelement_v4bf16_dynamic_sgpr(ptr addrspace(1)
;
; GFX1250-LABEL: v_insertelement_v4bf16_dynamic_sgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1311,6 +1351,8 @@ define amdgpu_kernel void @v_insertelement_v8bf16_3(ptr addrspace(1) %out, ptr a
;
; GFX1250-FAKE16-LABEL: v_insertelement_v8bf16_3:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1325,6 +1367,8 @@ define amdgpu_kernel void @v_insertelement_v8bf16_3(ptr addrspace(1) %out, ptr a
;
; GFX1250-REAL16-LABEL: v_insertelement_v8bf16_3:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1555,6 +1599,8 @@ define amdgpu_kernel void @v_insertelement_v8bf16_dynamic(ptr addrspace(1) %out,
;
; GFX1250-FAKE16-LABEL: v_insertelement_v8bf16_dynamic:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1600,6 +1646,8 @@ define amdgpu_kernel void @v_insertelement_v8bf16_dynamic(ptr addrspace(1) %out,
;
; GFX1250-REAL16-LABEL: v_insertelement_v8bf16_dynamic:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1736,6 +1784,8 @@ define amdgpu_kernel void @v_insertelement_v16bf16_3(ptr addrspace(1) %out, ptr
;
; GFX1250-FAKE16-LABEL: v_insertelement_v16bf16_3:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1757,6 +1807,8 @@ define amdgpu_kernel void @v_insertelement_v16bf16_3(ptr addrspace(1) %out, ptr
;
; GFX1250-REAL16-LABEL: v_insertelement_v16bf16_3:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2150,6 +2202,8 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
;
; GFX1250-FAKE16-LABEL: v_insertelement_v16bf16_dynamic:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2231,6 +2285,8 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
;
; GFX1250-REAL16-LABEL: v_insertelement_v16bf16_dynamic:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/intrinsic-amdgcn-s-alloc-vgpr.ll b/llvm/test/CodeGen/AMDGPU/intrinsic-amdgcn-s-alloc-vgpr.ll
index bf0a8acdc2cb8..70857c9635906 100644
--- a/llvm/test/CodeGen/AMDGPU/intrinsic-amdgcn-s-alloc-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/intrinsic-amdgcn-s-alloc-vgpr.ll
@@ -8,6 +8,8 @@ declare i1 @llvm.amdgcn.s.alloc.vgpr(i32)
define amdgpu_cs void @test_alloc_vreg_const(ptr addrspace(1) %out) #0 {
; GISEL-LABEL: test_alloc_vreg_const:
; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GISEL-NEXT: s_cmp_lg_u32 0, s33
@@ -26,6 +28,8 @@ define amdgpu_cs void @test_alloc_vreg_const(ptr addrspace(1) %out) #0 {
;
; DAGISEL-LABEL: test_alloc_vreg_const:
; DAGISEL: ; %bb.0: ; %entry
+; DAGISEL-NEXT: global_wb
+; DAGISEL-NEXT: v_nop
; DAGISEL-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; DAGISEL-NEXT: s_cmp_lg_u32 0, s33
@@ -40,6 +44,8 @@ define amdgpu_cs void @test_alloc_vreg_const(ptr addrspace(1) %out) #0 {
;
; NRBS-LABEL: test_alloc_vreg_const:
; NRBS: ; %bb.0: ; %entry
+; NRBS-NEXT: global_wb
+; NRBS-NEXT: v_nop
; NRBS-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; NRBS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; NRBS-NEXT: s_cmp_lg_u32 0, s33
@@ -65,6 +71,8 @@ entry:
define amdgpu_cs void @test_alloc_vreg_var(i32 inreg %n, ptr addrspace(1) %out) #0 {
; GISEL-LABEL: test_alloc_vreg_var:
; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GISEL-NEXT: s_cmp_lg_u32 0, s33
@@ -83,6 +91,8 @@ define amdgpu_cs void @test_alloc_vreg_var(i32 inreg %n, ptr addrspace(1) %out)
;
; DAGISEL-LABEL: test_alloc_vreg_var:
; DAGISEL: ; %bb.0: ; %entry
+; DAGISEL-NEXT: global_wb
+; DAGISEL-NEXT: v_nop
; DAGISEL-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; DAGISEL-NEXT: s_cmp_lg_u32 0, s33
@@ -97,6 +107,8 @@ define amdgpu_cs void @test_alloc_vreg_var(i32 inreg %n, ptr addrspace(1) %out)
;
; NRBS-LABEL: test_alloc_vreg_var:
; NRBS: ; %bb.0: ; %entry
+; NRBS-NEXT: global_wb
+; NRBS-NEXT: v_nop
; NRBS-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; NRBS-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; NRBS-NEXT: s_cmp_lg_u32 0, s33
@@ -122,6 +134,8 @@ entry:
define amdgpu_cs void @test_alloc_vreg_vgpr(i32 %n, ptr addrspace(1) %out) #0 {
; GISEL-LABEL: test_alloc_vreg_vgpr:
; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-NEXT: s_cmp_lg_u32 0, s33
@@ -143,6 +157,8 @@ define amdgpu_cs void @test_alloc_vreg_vgpr(i32 %n, ptr addrspace(1) %out) #0 {
;
; DAGISEL-LABEL: test_alloc_vreg_vgpr:
; DAGISEL: ; %bb.0: ; %entry
+; DAGISEL-NEXT: global_wb
+; DAGISEL-NEXT: v_nop
; DAGISEL-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; DAGISEL-NEXT: s_cmp_lg_u32 0, s33
@@ -160,6 +176,8 @@ define amdgpu_cs void @test_alloc_vreg_vgpr(i32 %n, ptr addrspace(1) %out) #0 {
;
; NRBS-LABEL: test_alloc_vreg_vgpr:
; NRBS: ; %bb.0: ; %entry
+; NRBS-NEXT: global_wb
+; NRBS-NEXT: v_nop
; NRBS-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
; NRBS-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; NRBS-NEXT: s_cmp_lg_u32 0, s33
diff --git a/llvm/test/CodeGen/AMDGPU/literal64.ll b/llvm/test/CodeGen/AMDGPU/literal64.ll
index 260ee0701ff3a..e4d3844e07bb2 100644
--- a/llvm/test/CodeGen/AMDGPU/literal64.ll
+++ b/llvm/test/CodeGen/AMDGPU/literal64.ll
@@ -7,6 +7,8 @@
define amdgpu_ps i64 @s_add_u64(i64 inreg %a) {
; GFX1250-LABEL: s_add_u64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0xf12345678
; GFX1250-NEXT: ; return to shader part epilog
@@ -22,6 +24,8 @@ define amdgpu_ps i64 @s_add_u64(i64 inreg %a) {
define amdgpu_ps void @v_add_u64(i64 %a, ptr addrspace(1) %out) {
; GFX1250-LABEL: v_add_u64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 0xf12345678, v[0:1]
; GFX1250-NEXT: global_store_b64 v[2:3], v[0:1], off
@@ -42,6 +46,8 @@ define amdgpu_ps void @v_add_u64(i64 %a, ptr addrspace(1) %out) {
define amdgpu_ps i64 @s_add_neg_u64(i64 inreg %a) {
; GFX1250-LABEL: s_add_neg_u64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0xfffffff0edcba988
; GFX1250-NEXT: ; return to shader part epilog
@@ -57,6 +63,8 @@ define amdgpu_ps i64 @s_add_neg_u64(i64 inreg %a) {
define amdgpu_ps void @v_add_neg_u64(i64 %a, ptr addrspace(1) %out) {
; GFX1250-LABEL: v_add_neg_u64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 0xfffffff0edcba988, v[0:1]
; GFX1250-NEXT: global_store_b64 v[2:3], v[0:1], off
@@ -77,6 +85,8 @@ define amdgpu_ps void @v_add_neg_u64(i64 %a, ptr addrspace(1) %out) {
define amdgpu_ps i64 @s_sub_u64(i64 inreg %a) {
; GFX1250-LABEL: s_sub_u64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sub_nc_u64 s[0:1], 0xf12345678, s[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -92,6 +102,8 @@ define amdgpu_ps i64 @s_sub_u64(i64 inreg %a) {
define amdgpu_ps void @v_sub_u64(i64 %a, ptr addrspace(1) %out) {
; GFX1250-LABEL: v_sub_u64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], 0xf12345678, v[0:1]
; GFX1250-NEXT: global_store_b64 v[2:3], v[0:1], off
@@ -280,6 +292,8 @@ define double @rsq_f64() {
define amdgpu_ps i64 @s_and_b64(i64 inreg %a) {
; GFX1250-LABEL: s_and_b64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b64 s[0:1], s[0:1], 0xf12345678
; GFX1250-NEXT: ; return to shader part epilog
@@ -297,6 +311,8 @@ define amdgpu_ps i64 @s_and_b64(i64 inreg %a) {
define amdgpu_ps void @v_and_b64(i64 %a, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: v_and_b64:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 15, v1
; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x12345678, v0
@@ -305,6 +321,8 @@ define amdgpu_ps void @v_and_b64(i64 %a, ptr addrspace(1) %out) {
;
; GFX1250-GISEL-LABEL: v_and_b64:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x12345678, v0
; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 15, v1
@@ -332,6 +350,8 @@ define amdgpu_ps void @v_and_b64(i64 %a, ptr addrspace(1) %out) {
define amdgpu_ps <2 x float> @v_add_f64_200.1(double %a) {
; GFX1250-LABEL: v_add_f64_200.1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_f64_e32 v[0:1], 0x4069033333333333, v[0:1]
@@ -351,6 +371,8 @@ define amdgpu_ps <2 x float> @v_add_f64_200.1(double %a) {
define amdgpu_ps <2 x float> @v_add_f64_200.0(double %a) {
; GFX1250-LABEL: v_add_f64_200.0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_f64_e32 v[0:1], 0x40690000, v[0:1]
@@ -370,6 +392,8 @@ define amdgpu_ps <2 x float> @v_add_f64_200.0(double %a) {
define amdgpu_ps <2 x float> @v_lshl_add_u64(i64 %a) {
; GFX1250-SDAG-LABEL: v_lshl_add_u64:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xf12345678
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -378,6 +402,8 @@ define amdgpu_ps <2 x float> @v_lshl_add_u64(i64 %a) {
;
; GFX1250-GISEL-LABEL: v_lshl_add_u64:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], 0xf12345678
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -402,6 +428,8 @@ define amdgpu_ps <2 x float> @v_lshl_add_u64(i64 %a) {
define amdgpu_ps <2 x float> @v_fma_f64(double %a, double %b) {
; GFX1250-SDAG-LABEL: v_fma_f64:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_fmaak_f64 v[4:5], v[0:1], v[2:3], 0x4063233333333333
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x4069033333333333
@@ -414,6 +442,8 @@ define amdgpu_ps <2 x float> @v_fma_f64(double %a, double %b) {
;
; GFX1250-GISEL-LABEL: v_fma_f64:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], 0x4063233333333333
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
@@ -460,6 +490,8 @@ define amdgpu_ps <2 x float> @v_fma_f64(double %a, double %b) {
define amdgpu_ps <2 x float> @v_add_neg_f64(double %a) {
; GFX1250-SDAG-LABEL: v_add_neg_f64:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0x4069033333333333
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -468,6 +500,8 @@ define amdgpu_ps <2 x float> @v_add_neg_f64(double %a) {
;
; GFX1250-GISEL-LABEL: v_add_neg_f64:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], 0x4069033333333333
@@ -500,6 +534,8 @@ define amdgpu_ps <2 x float> @v_add_neg_f64(double %a) {
define amdgpu_ps <2 x float> @v_cndmask(double %a) {
; GFX1250-SDAG-LABEL: v_cndmask:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cmp_eq_f64_e32 vcc_lo, 0, v[0:1]
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0x40632000
@@ -510,6 +546,8 @@ define amdgpu_ps <2 x float> @v_cndmask(double %a) {
;
; GFX1250-GISEL-LABEL: v_cndmask:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cmp_eq_f64_e32 vcc_lo, 0, v[0:1]
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0x40690333
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
index c418d9e95050a..da20456b3c07a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
@@ -1240,6 +1240,8 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
;
; GFX1250-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_clause 0x1
@@ -1325,6 +1327,8 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
;
; GFX1250-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: global_wb
+; GFX1250-ISEL-NEXT: v_nop
; GFX1250-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
@@ -1419,6 +1423,8 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
;
; GFX1250-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_clause 0x1
@@ -1504,6 +1510,8 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
;
; GFX1250-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-ISEL: ; %bb.0:
+; GFX1250-ISEL-NEXT: global_wb
+; GFX1250-ISEL-NEXT: v_nop
; GFX1250-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
index 7fd5b35783f30..07e820b08c00b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
@@ -17,6 +17,8 @@ define amdgpu_ps float @bitop3_b32_vvv(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: bitop3_b32_vvv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, v2 bitop3:0xf
; GFX1250-NEXT: ; return to shader part epilog
@@ -33,6 +35,8 @@ define amdgpu_ps float @bitop3_b32_svv(i32 inreg %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: bitop3_b32_svv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, s0, v0, v1 bitop3:0x10
; GFX1250-NEXT: ; return to shader part epilog
@@ -50,6 +54,8 @@ define amdgpu_ps float @bitop3_b32_ssv(i32 inreg %a, i32 inreg %b, i32 %c) {
;
; GFX1250-LABEL: bitop3_b32_ssv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, s0, s1, v0 bitop3:0x11
; GFX1250-NEXT: ; return to shader part epilog
@@ -68,6 +74,8 @@ define amdgpu_ps float @bitop3_b32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c)
;
; GFX1250-LABEL: bitop3_b32_sss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -93,6 +101,8 @@ define amdgpu_ps float @bitop3_b32_vvi(i32 %a, i32 %b) {
;
; GFX1250-LABEL: bitop3_b32_vvi:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_bitop3_b32 v0, v0, v1, 0x3e8 bitop3:0x13
; GFX1250-NEXT: ; return to shader part epilog
@@ -118,6 +128,8 @@ define amdgpu_ps float @bitop3_b32_vii(i32 %a) {
;
; GFX1250-SDAG-LABEL: bitop3_b32_vii:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_movk_i32 s0, 0x7d0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -126,6 +138,8 @@ define amdgpu_ps float @bitop3_b32_vii(i32 %a) {
;
; GFX1250-GISEL-LABEL: bitop3_b32_vii:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e8
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -157,6 +171,8 @@ define amdgpu_ps float @bitop3_b32_iii() {
;
; GFX1250-SDAG-LABEL: bitop3_b32_iii:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0x3e8
; GFX1250-SDAG-NEXT: s_movk_i32 s0, 0xbb8
@@ -166,6 +182,8 @@ define amdgpu_ps float @bitop3_b32_iii() {
;
; GFX1250-GISEL-LABEL: bitop3_b32_iii:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0x7d0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0x3e8
@@ -185,12 +203,16 @@ define amdgpu_ps half @bitop3_b16_vvv(i16 %a, i16 %b, i16 %c) {
;
; GFX1250-TRUE16-LABEL: bitop3_b16_vvv:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, v2.l bitop3:0xf
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-FAKE16-LABEL: bitop3_b16_vvv:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, v2 bitop3:0xf
; GFX1250-FAKE16-NEXT: ; return to shader part epilog
@@ -207,12 +229,16 @@ define amdgpu_ps half @bitop3_b16_svv(i16 inreg %a, i16 %b, i16 %c) {
;
; GFX1250-TRUE16-LABEL: bitop3_b16_svv:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, v0.l, v1.l bitop3:0x10
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-FAKE16-LABEL: bitop3_b16_svv:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_bitop3_b16 v0, s0, v0, v1 bitop3:0x10
; GFX1250-FAKE16-NEXT: ; return to shader part epilog
@@ -230,12 +256,16 @@ define amdgpu_ps half @bitop3_b16_ssv(i16 inreg %a, i16 inreg %b, i16 %c) {
;
; GFX1250-TRUE16-LABEL: bitop3_b16_ssv:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x11
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-FAKE16-LABEL: bitop3_b16_ssv:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_bitop3_b16 v0, s0, s1, v0 bitop3:0x11
; GFX1250-FAKE16-NEXT: ; return to shader part epilog
@@ -254,6 +284,8 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
;
; GFX1250-TRUE16-LABEL: bitop3_b16_sss:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -262,6 +294,8 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
;
; GFX1250-FAKE16-LABEL: bitop3_b16_sss:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -287,12 +321,16 @@ define amdgpu_ps half @bitop3_b16_vvi(i16 %a, i16 %b) {
;
; GFX1250-TRUE16-LABEL: bitop3_b16_vvi:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0x3e8 bitop3:0x13
; GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-FAKE16-LABEL: bitop3_b16_vvi:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_bitop3_b16 v0, v0, v1, 0x3e8 bitop3:0x13
; GFX1250-FAKE16-NEXT: ; return to shader part epilog
@@ -318,6 +356,8 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
;
; GFX1250-SDG-TRUE16-LABEL: bitop3_b16_vii:
; GFX1250-SDG-TRUE16: ; %bb.0:
+; GFX1250-SDG-TRUE16-NEXT: global_wb
+; GFX1250-SDG-TRUE16-NEXT: v_nop
; GFX1250-SDG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x7d0
; GFX1250-SDG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -326,6 +366,8 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
;
; GFX1250-SDG-FAKE16-LABEL: bitop3_b16_vii:
; GFX1250-SDG-FAKE16: ; %bb.0:
+; GFX1250-SDG-FAKE16-NEXT: global_wb
+; GFX1250-SDG-FAKE16-NEXT: v_nop
; GFX1250-SDG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDG-FAKE16-NEXT: s_movk_i32 s0, 0x7d0
; GFX1250-SDG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -334,6 +376,8 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
;
; GFX1250-GISEL-TRUE16-LABEL: bitop3_b16_vii:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3e8
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -342,6 +386,8 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: bitop3_b16_vii:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e8
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -372,6 +418,8 @@ define amdgpu_ps half @bitop3_b16_iii() {
;
; GFX1250-SDG-TRUE16-LABEL: bitop3_b16_iii:
; GFX1250-SDG-TRUE16: ; %bb.0:
+; GFX1250-SDG-TRUE16-NEXT: global_wb
+; GFX1250-SDG-TRUE16-NEXT: v_nop
; GFX1250-SDG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x7d0
; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xbb8
@@ -381,6 +429,8 @@ define amdgpu_ps half @bitop3_b16_iii() {
;
; GFX1250-SDG-FAKE16-LABEL: bitop3_b16_iii:
; GFX1250-SDG-FAKE16: ; %bb.0:
+; GFX1250-SDG-FAKE16-NEXT: global_wb
+; GFX1250-SDG-FAKE16-NEXT: v_nop
; GFX1250-SDG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDG-FAKE16-NEXT: v_mov_b32_e32 v0, 0x3e8
; GFX1250-SDG-FAKE16-NEXT: s_movk_i32 s0, 0xbb8
@@ -390,6 +440,8 @@ define amdgpu_ps half @bitop3_b16_iii() {
;
; GFX1250-GISEL-TRUE16-LABEL: bitop3_b16_iii:
; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: global_wb
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x7d0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3e8
@@ -399,6 +451,8 @@ define amdgpu_ps half @bitop3_b16_iii() {
;
; GFX1250-GISEL-FAKE16-LABEL: bitop3_b16_iii:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, 0x7d0
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0x3e8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.id.ll
index 3a8c10e801a85..c325c877cdd56 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.id.ll
@@ -11,6 +11,8 @@ declare i32 @llvm.amdgcn.cluster.id.z() #0
define amdgpu_kernel void @test_cluster_id_x(ptr addrspace(1) %out) {
; CHECK-UNKNOWN-LABEL: test_cluster_id_x:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[2:3], s[0:1], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_dual_mov_b32 v0, ttmp9 :: v_dual_mov_b32 v1, 0
@@ -88,6 +90,8 @@ define amdgpu_kernel void @test_cluster_id_x(ptr addrspace(1) %out) {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv
; CHECK-MESA3D-NEXT: v_dual_mov_b32 v0, ttmp9 :: v_dual_mov_b32 v1, 0
@@ -97,6 +101,8 @@ define amdgpu_kernel void @test_cluster_id_x(ptr addrspace(1) %out) {
;
; CHECK-G-UNKNOWN-LABEL: test_cluster_id_x:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[2:3], s[0:1], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_dual_mov_b32 v0, ttmp9 :: v_dual_mov_b32 v1, 0
@@ -174,6 +180,8 @@ define amdgpu_kernel void @test_cluster_id_x(ptr addrspace(1) %out) {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_dual_mov_b32 v0, ttmp9 :: v_dual_mov_b32 v1, 0
@@ -188,6 +196,8 @@ define amdgpu_kernel void @test_cluster_id_x(ptr addrspace(1) %out) {
define amdgpu_kernel void @test_cluster_id_y(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_cluster_id_y:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[2:3], s[0:1], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_dual_mov_b32 v0, ttmp7 :: v_dual_mov_b32 v1, 0
@@ -265,6 +275,8 @@ define amdgpu_kernel void @test_cluster_id_y(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv
; CHECK-MESA3D-NEXT: v_dual_mov_b32 v0, ttmp7 :: v_dual_mov_b32 v1, 0
@@ -274,6 +286,8 @@ define amdgpu_kernel void @test_cluster_id_y(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_cluster_id_y:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[2:3], s[0:1], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_dual_mov_b32 v0, ttmp7 :: v_dual_mov_b32 v1, 0
@@ -351,6 +365,8 @@ define amdgpu_kernel void @test_cluster_id_y(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_dual_mov_b32 v0, ttmp7 :: v_dual_mov_b32 v1, 0
@@ -365,6 +381,8 @@ define amdgpu_kernel void @test_cluster_id_y(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @test_cluster_id_z(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_cluster_id_z:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[2:3], s[0:1], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_wait_xcnt 0x0
@@ -445,6 +463,8 @@ define amdgpu_kernel void @test_cluster_id_z(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv
; CHECK-MESA3D-NEXT: s_lshr_b32 s2, ttmp7, 16
@@ -456,6 +476,8 @@ define amdgpu_kernel void @test_cluster_id_z(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_cluster_id_z:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[2:3], s[0:1], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_wait_xcnt 0x0
@@ -536,6 +558,8 @@ define amdgpu_kernel void @test_cluster_id_z(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_lshr_b32 s2, ttmp7, 16
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll
index 5e8fac91a20d6..113b306d2b91b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll
@@ -10,6 +10,8 @@ declare void @llvm.amdgcn.cluster.load.async.to.lds.b128(ptr addrspace(1) %gaddr
define amdgpu_ps void @cluster_load_async_to_lds_b8_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %mask) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b8_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
@@ -19,6 +21,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b8_vaddr(ptr addrspace(1) %gadd
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b8_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
@@ -36,6 +40,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b8_vaddr_imm_mask(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b8_vaddr_imm_mask:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: s_mov_b32 m0, 15
@@ -44,6 +50,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b8_vaddr_imm_mask(ptr addrspace
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b8_vaddr_imm_mask:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -60,6 +68,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b8_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 inreg %mask) {
; GFX1250-LABEL: cluster_load_async_to_lds_b8_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -74,6 +84,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b32_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %mask) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b32_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
@@ -83,6 +95,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b32_vaddr(ptr addrspace(1) %gad
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b32_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
@@ -100,6 +114,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b32_vaddr_imm_mask(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b32_vaddr_imm_mask:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: s_mov_b32 m0, 15
@@ -108,6 +124,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b32_vaddr_imm_mask(ptr addrspac
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b32_vaddr_imm_mask:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -124,6 +142,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b32_saddr( ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 inreg %mask) {
; GFX1250-LABEL: cluster_load_async_to_lds_b32_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -138,6 +158,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b64_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %mask) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b64_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
@@ -147,6 +169,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b64_vaddr(ptr addrspace(1) %gad
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b64_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
@@ -164,6 +188,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b64_vaddr_imm_mask( ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b64_vaddr_imm_mask:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: s_movk_i32 m0, 0x7f
@@ -172,6 +198,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b64_vaddr_imm_mask( ptr addrspa
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b64_vaddr_imm_mask:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -188,6 +216,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b64_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 inreg %mask) {
; GFX1250-LABEL: cluster_load_async_to_lds_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -202,6 +232,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b128_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr, i32 %mask) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b128_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
@@ -211,6 +243,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b128_vaddr(ptr addrspace(1) %ga
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b128_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v3
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
@@ -228,6 +262,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b128_vaddr_imm_mask(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b128_vaddr_imm_mask:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: s_movk_i32 m0, 0x7f
@@ -236,6 +272,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b128_vaddr_imm_mask(ptr addrspa
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b128_vaddr_imm_mask:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -252,6 +290,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b128_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 inreg %mask) {
; GFX1250-LABEL: cluster_load_async_to_lds_b128_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -266,6 +306,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b32_saddr_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 inreg %mask, i32 %idx) {
; GFX1250-LABEL: cluster_load_async_to_lds_b32_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 m0, s2
; GFX1250-NEXT: cluster_load_async_to_lds_b32 v0, v1, s[0:1] offset:16 scale_offset th:TH_LOAD_NT
@@ -280,6 +322,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b64_saddr_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 inreg %mask, i32 %idx) {
; GFX1250-LABEL: cluster_load_async_to_lds_b64_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 m0, s2
; GFX1250-NEXT: cluster_load_async_to_lds_b64 v0, v1, s[0:1] offset:16 scale_offset th:TH_LOAD_NT
@@ -294,6 +338,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b64_saddr_no_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 inreg %mask, i32 %idx) {
; GFX1250-LABEL: cluster_load_async_to_lds_b64_saddr_no_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -312,6 +358,8 @@ entry:
define amdgpu_ps void @cluster_load_async_to_lds_b32_saddr_sladdr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) inreg %laddr, i32 inreg %mask) {
; GFX1250-SDAG-LABEL: cluster_load_async_to_lds_b32_saddr_sladdr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, 32 :: v_dual_mov_b32 v1, s2
; GFX1250-SDAG-NEXT: s_mov_b32 m0, s3
@@ -320,6 +368,8 @@ define amdgpu_ps void @cluster_load_async_to_lds_b32_saddr_sladdr(ptr addrspace(
;
; GFX1250-GISEL-LABEL: cluster_load_async_to_lds_b32_saddr_sladdr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 32
; GFX1250-GISEL-NEXT: s_mov_b32 m0, s3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.ll
index a7329f923ca28..cfc80c9ea8e2a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.ll
@@ -9,6 +9,8 @@ declare <4 x i32> @llvm.amdgcn.cluster.load.b128.v4i32.p1(ptr addrspace(1), i32
define amdgpu_ps void @cluster_load_b32_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use, i32 %mask) {
; GFX1250-LABEL: cluster_load_b32_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v4
; GFX1250-NEXT: s_mov_b32 m0, s0
@@ -26,6 +28,8 @@ entry:
define amdgpu_ps void @cluster_load_b32_vaddr_imm_mask(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: cluster_load_b32_vaddr_imm_mask:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 m0, 7
; GFX1250-NEXT: cluster_load_b32 v0, v[0:1], off offset:32 th:TH_LOAD_HT scope:SCOPE_SE
@@ -42,6 +46,8 @@ entry:
define amdgpu_ps void @cluster_load_b32_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 inreg %mask) {
; GFX1250-LABEL: cluster_load_b32_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -59,6 +65,8 @@ entry:
define amdgpu_ps void @cluster_load_monitor_b32_saddr_scale_offset(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 inreg %mask, i32 %idx) {
; GFX1250-LABEL: cluster_load_monitor_b32_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 m0, s2
; GFX1250-NEXT: cluster_load_b32 v2, v2, s[0:1] scale_offset th:TH_LOAD_BYPASS scope:SCOPE_SYS
@@ -76,6 +84,8 @@ entry:
define amdgpu_ps void @cluster_load_b64_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use, i32 %mask) {
; GFX1250-LABEL: cluster_load_b64_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v4
; GFX1250-NEXT: s_mov_b32 m0, s0
@@ -93,6 +103,8 @@ entry:
define amdgpu_ps void @cluster_load_b64_vaddr_imm_mask(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: cluster_load_b64_vaddr_imm_mask:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 m0, 0x10007
; GFX1250-NEXT: cluster_load_b64 v[0:1], v[0:1], off offset:32 th:TH_LOAD_HT scope:SCOPE_SE
@@ -109,6 +121,8 @@ entry:
define amdgpu_ps void @cluster_load_b64_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 inreg %mask) {
; GFX1250-LABEL: cluster_load_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -126,6 +140,8 @@ entry:
define amdgpu_ps void @cluster_load_monitor_b64_saddr_scale_offset(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 inreg %mask, i32 %idx) {
; GFX1250-LABEL: cluster_load_monitor_b64_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 m0, s2
; GFX1250-NEXT: cluster_load_b64 v[2:3], v2, s[0:1] scale_offset th:TH_LOAD_BYPASS scope:SCOPE_SYS
@@ -143,6 +159,8 @@ entry:
define amdgpu_ps void @cluster_load_b128_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use, i32 %mask) {
; GFX1250-LABEL: cluster_load_b128_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v4
; GFX1250-NEXT: s_mov_b32 m0, s0
@@ -160,6 +178,8 @@ entry:
define amdgpu_ps void @cluster_load_b128_vaddr_imm_mask(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: cluster_load_b128_vaddr_imm_mask:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 m0, 15
; GFX1250-NEXT: cluster_load_b128 v[4:7], v[0:1], off offset:32 th:TH_LOAD_HT scope:SCOPE_SE
@@ -176,6 +196,8 @@ entry:
define amdgpu_ps void @cluster_load_b128_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 inreg %mask) {
; GFX1250-LABEL: cluster_load_b128_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_mov_b32 m0, s2
@@ -192,6 +214,8 @@ entry:
define amdgpu_ps void @cluster_load_b32_saddr_vmask(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 %mask) {
; GFX1250-LABEL: cluster_load_b32_saddr_vmask:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v2
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -210,6 +234,8 @@ entry:
define amdgpu_ps void @cluster_load_b64_saddr_vmask(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 %mask) {
; GFX1250-LABEL: cluster_load_b64_saddr_vmask:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v2
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -228,6 +254,8 @@ entry:
define amdgpu_ps void @cluster_load_b128_saddr_vmask(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 %mask) {
; GFX1250-LABEL: cluster_load_b128_saddr_vmask:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v2
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.id.ll
index 16a9a9b9d107c..f240de6a68bf6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.id.ll
@@ -11,6 +11,8 @@ declare i32 @llvm.amdgcn.cluster.workgroup.id.z() #0
define amdgpu_kernel void @test_workgroup_id_x(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_workgroup_id_x:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_and_b32 s2, ttmp6, 15
@@ -90,6 +92,8 @@ define amdgpu_kernel void @test_workgroup_id_x(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_and_b32 s2, ttmp6, 15
@@ -101,6 +105,8 @@ define amdgpu_kernel void @test_workgroup_id_x(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_id_x:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_and_b32 s2, ttmp6, 15
@@ -180,6 +186,8 @@ define amdgpu_kernel void @test_workgroup_id_x(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_and_b32 s2, ttmp6, 15
@@ -196,6 +204,8 @@ define amdgpu_kernel void @test_workgroup_id_x(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @test_workgroup_id_x_optimized(ptr addrspace(1) %out) "amdgpu-cluster-dims"="1,2,2" {
; CHECK-UNKNOWN-LABEL: test_workgroup_id_x_optimized:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_mov_b32_e32 v0, 0
@@ -273,6 +283,8 @@ define amdgpu_kernel void @test_workgroup_id_x_optimized(ptr addrspace(1) %out)
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: v_mov_b32_e32 v0, 0
@@ -282,6 +294,8 @@ define amdgpu_kernel void @test_workgroup_id_x_optimized(ptr addrspace(1) %out)
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_id_x_optimized:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_mov_b32_e32 v0, 0
@@ -359,6 +373,8 @@ define amdgpu_kernel void @test_workgroup_id_x_optimized(ptr addrspace(1) %out)
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_mov_b32_e32 v0, 0
@@ -373,6 +389,8 @@ define amdgpu_kernel void @test_workgroup_id_x_optimized(ptr addrspace(1) %out)
define amdgpu_kernel void @test_workgroup_id_y(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_workgroup_id_y:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40004
@@ -452,6 +470,8 @@ define amdgpu_kernel void @test_workgroup_id_y(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40004
@@ -463,6 +483,8 @@ define amdgpu_kernel void @test_workgroup_id_y(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_id_y:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40004
@@ -542,6 +564,8 @@ define amdgpu_kernel void @test_workgroup_id_y(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40004
@@ -558,6 +582,8 @@ define amdgpu_kernel void @test_workgroup_id_y(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @test_workgroup_id_y_optimized(ptr addrspace(1) %out) "amdgpu-cluster-dims"="2,1,2" {
; CHECK-UNKNOWN-LABEL: test_workgroup_id_y_optimized:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_mov_b32_e32 v0, 0
@@ -635,6 +661,8 @@ define amdgpu_kernel void @test_workgroup_id_y_optimized(ptr addrspace(1) %out)
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: v_mov_b32_e32 v0, 0
@@ -644,6 +672,8 @@ define amdgpu_kernel void @test_workgroup_id_y_optimized(ptr addrspace(1) %out)
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_id_y_optimized:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_mov_b32_e32 v0, 0
@@ -721,6 +751,8 @@ define amdgpu_kernel void @test_workgroup_id_y_optimized(ptr addrspace(1) %out)
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_mov_b32_e32 v0, 0
@@ -735,6 +767,8 @@ define amdgpu_kernel void @test_workgroup_id_y_optimized(ptr addrspace(1) %out)
define amdgpu_kernel void @test_workgroup_id_z(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_workgroup_id_z:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40008
@@ -814,6 +848,8 @@ define amdgpu_kernel void @test_workgroup_id_z(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40008
@@ -825,6 +861,8 @@ define amdgpu_kernel void @test_workgroup_id_z(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_id_z:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40008
@@ -904,6 +942,8 @@ define amdgpu_kernel void @test_workgroup_id_z(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40008
@@ -920,6 +960,8 @@ define amdgpu_kernel void @test_workgroup_id_z(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @test_workgroup_flat_id(ptr addrspace(1) %out) {
; CHECK-UNKNOWN-LABEL: test_workgroup_flat_id:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_getreg_b32 s2, hwreg(HW_REG_IB_STS2, 21, 4)
@@ -999,6 +1041,8 @@ define amdgpu_kernel void @test_workgroup_flat_id(ptr addrspace(1) %out) {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_getreg_b32 s2, hwreg(HW_REG_IB_STS2, 21, 4)
@@ -1010,6 +1054,8 @@ define amdgpu_kernel void @test_workgroup_flat_id(ptr addrspace(1) %out) {
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_flat_id:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_getreg_b32 s2, hwreg(HW_REG_IB_STS2, 21, 4)
@@ -1089,6 +1135,8 @@ define amdgpu_kernel void @test_workgroup_flat_id(ptr addrspace(1) %out) {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_getreg_b32 s2, hwreg(HW_REG_IB_STS2, 21, 4)
@@ -1105,6 +1153,8 @@ define amdgpu_kernel void @test_workgroup_flat_id(ptr addrspace(1) %out) {
define amdgpu_kernel void @test_workgroup_id_z_optimized(ptr addrspace(1) %out) "amdgpu-cluster-dims"="2,2,1" {
; CHECK-UNKNOWN-LABEL: test_workgroup_id_z_optimized:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_mov_b32_e32 v0, 0
@@ -1182,6 +1232,8 @@ define amdgpu_kernel void @test_workgroup_id_z_optimized(ptr addrspace(1) %out)
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: v_mov_b32_e32 v0, 0
@@ -1191,6 +1243,8 @@ define amdgpu_kernel void @test_workgroup_id_z_optimized(ptr addrspace(1) %out)
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_id_z_optimized:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_mov_b32_e32 v0, 0
@@ -1268,6 +1322,8 @@ define amdgpu_kernel void @test_workgroup_id_z_optimized(ptr addrspace(1) %out)
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.flat.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.flat.id.ll
index e61ef263fc14c..35e70eb8f5551 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.flat.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.flat.id.ll
@@ -9,6 +9,8 @@ declare i32 @llvm.amdgcn.cluster.workgroup.max.flat.id() #0
define amdgpu_kernel void @test_workgroup_max_flat_id(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_workgroup_max_flat_id:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40018
@@ -88,6 +90,8 @@ define amdgpu_kernel void @test_workgroup_max_flat_id(ptr addrspace(1) %out) #1
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40018
@@ -99,6 +103,8 @@ define amdgpu_kernel void @test_workgroup_max_flat_id(ptr addrspace(1) %out) #1
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_max_flat_id:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40018
@@ -178,6 +184,8 @@ define amdgpu_kernel void @test_workgroup_max_flat_id(ptr addrspace(1) %out) #1
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40018
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.id.ll
index 83e00e7d4bbf7..536bfef3568c5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.id.ll
@@ -11,6 +11,8 @@ declare i32 @llvm.amdgcn.cluster.workgroup.max.id.z() #0
define amdgpu_kernel void @test_workgroup_max_id_x(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_workgroup_max_id_x:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x4000c
@@ -90,6 +92,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x4000c
@@ -101,6 +105,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_max_id_x:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x4000c
@@ -180,6 +186,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x4000c
@@ -196,6 +204,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @test_workgroup_max_id_x_optimized(ptr addrspace(1) %out) "amdgpu-cluster-dims"="5,6,7" {
; CHECK-UNKNOWN-LABEL: test_workgroup_max_id_x_optimized:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4
@@ -273,6 +283,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x_optimized(ptr addrspace(1) %o
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4
@@ -282,6 +294,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x_optimized(ptr addrspace(1) %o
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_max_id_x_optimized:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_dual_mov_b32 v0, 4 :: v_dual_mov_b32 v1, 0
@@ -359,6 +373,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x_optimized(ptr addrspace(1) %o
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_dual_mov_b32 v0, 4 :: v_dual_mov_b32 v1, 0
@@ -373,6 +389,8 @@ define amdgpu_kernel void @test_workgroup_max_id_x_optimized(ptr addrspace(1) %o
define amdgpu_kernel void @test_workgroup_max_id_y(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_workgroup_max_id_y:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40010
@@ -452,6 +470,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40010
@@ -463,6 +483,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_max_id_y:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40010
@@ -542,6 +564,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40010
@@ -558,6 +582,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @test_workgroup_max_id_y_optimized(ptr addrspace(1) %out) "amdgpu-cluster-dims"="5,6,7" {
; CHECK-UNKNOWN-LABEL: test_workgroup_max_id_y_optimized:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 5
@@ -635,6 +661,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y_optimized(ptr addrspace(1) %o
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 5
@@ -644,6 +672,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y_optimized(ptr addrspace(1) %o
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_max_id_y_optimized:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_dual_mov_b32 v0, 5 :: v_dual_mov_b32 v1, 0
@@ -721,6 +751,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y_optimized(ptr addrspace(1) %o
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_dual_mov_b32 v0, 5 :: v_dual_mov_b32 v1, 0
@@ -735,6 +767,8 @@ define amdgpu_kernel void @test_workgroup_max_id_y_optimized(ptr addrspace(1) %o
define amdgpu_kernel void @test_workgroup_max_id_z(ptr addrspace(1) %out) #1 {
; CHECK-UNKNOWN-LABEL: test_workgroup_max_id_z:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40014
@@ -814,6 +848,8 @@ define amdgpu_kernel void @test_workgroup_max_id_z(ptr addrspace(1) %out) #1 {
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40014
@@ -825,6 +861,8 @@ define amdgpu_kernel void @test_workgroup_max_id_z(ptr addrspace(1) %out) #1 {
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_max_id_z:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: s_bfe_u32 s2, ttmp6, 0x40014
@@ -904,6 +942,8 @@ define amdgpu_kernel void @test_workgroup_max_id_z(ptr addrspace(1) %out) #1 {
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: s_bfe_u32 s2, ttmp6, 0x40014
@@ -920,6 +960,8 @@ define amdgpu_kernel void @test_workgroup_max_id_z(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @test_workgroup_max_id_z_optimized(ptr addrspace(1) %out) "amdgpu-cluster-dims"="5,6,7" {
; CHECK-UNKNOWN-LABEL: test_workgroup_max_id_z_optimized:
; CHECK-UNKNOWN: ; %bb.0:
+; CHECK-UNKNOWN-NEXT: global_wb
+; CHECK-UNKNOWN-NEXT: v_nop
; CHECK-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-UNKNOWN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 6
@@ -997,6 +1039,8 @@ define amdgpu_kernel void @test_workgroup_max_id_z_optimized(ptr addrspace(1) %o
; CHECK-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-MESA3D-NEXT: ; %bb.0:
+; CHECK-MESA3D-NEXT: global_wb
+; CHECK-MESA3D-NEXT: v_nop
; CHECK-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-MESA3D-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 6
@@ -1006,6 +1050,8 @@ define amdgpu_kernel void @test_workgroup_max_id_z_optimized(ptr addrspace(1) %o
;
; CHECK-G-UNKNOWN-LABEL: test_workgroup_max_id_z_optimized:
; CHECK-G-UNKNOWN: ; %bb.0:
+; CHECK-G-UNKNOWN-NEXT: global_wb
+; CHECK-G-UNKNOWN-NEXT: v_nop
; CHECK-G-UNKNOWN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-UNKNOWN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; CHECK-G-UNKNOWN-NEXT: v_dual_mov_b32 v0, 6 :: v_dual_mov_b32 v1, 0
@@ -1083,6 +1129,8 @@ define amdgpu_kernel void @test_workgroup_max_id_z_optimized(ptr addrspace(1) %o
; CHECK-G-MESA3D-NEXT: runtime_loader_kernel_symbol = 0
; CHECK-G-MESA3D-NEXT: .end_amd_kernel_code_t
; CHECK-G-MESA3D-NEXT: ; %bb.0:
+; CHECK-G-MESA3D-NEXT: global_wb
+; CHECK-G-MESA3D-NEXT: v_nop
; CHECK-G-MESA3D-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-G-MESA3D-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; CHECK-G-MESA3D-NEXT: v_dual_mov_b32 v0, 6 :: v_dual_mov_b32 v1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll
index b75531748758b..db53a2593d5fa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.amdgcn.cos.bf16(bfloat) #0
define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-LABEL: cos_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -21,6 +23,8 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; REAL16-LABEL: cos_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -36,6 +40,8 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @cos_bf16_constant_4(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: cos_bf16_constant_4:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_cos_bf16_e32 v0, 4.0
@@ -46,6 +52,8 @@ define amdgpu_kernel void @cos_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: cos_bf16_constant_4:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_cos_bf16_e32 v0.l, 4.0
@@ -61,6 +69,8 @@ define amdgpu_kernel void @cos_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @cos_bf16_constant_100(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: cos_bf16_constant_100:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_cos_bf16_e32 v0, 0x42c8
@@ -71,6 +81,8 @@ define amdgpu_kernel void @cos_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: cos_bf16_constant_100:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_cos_bf16_e32 v0.l, 0x42c8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
index 2b0d33bcba978..7d715b48b60b4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
@@ -12,6 +12,8 @@ declare <2 x half> @llvm.amdgcn.cvt.pk.f16.fp8(i16)
define amdgpu_ps float @test_cvt_f16_bf8_byte0(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_bf8_byte0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -20,6 +22,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_bf8_byte0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_bf8_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -28,6 +32,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_bf8_byte0:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e32 v0.l, v0
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -36,6 +42,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_bf8_byte0:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_bf8_e32 v0, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -49,6 +57,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0(i32 %a) {
define amdgpu_ps float @test_cvt_f16_bf8_byte1(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_bf8_byte1:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.l, v0 byte_sel:1
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -57,6 +67,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte1(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_bf8_byte1:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:1
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -65,6 +77,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte1(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_bf8_byte1:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.l, v0 byte_sel:1
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -73,6 +87,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte1(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_bf8_byte1:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:1
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -86,6 +102,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte1(i32 %a) {
define amdgpu_ps float @test_cvt_f16_bf8_byte2(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_bf8_byte2:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.l, v0 byte_sel:2
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -94,6 +112,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte2(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_bf8_byte2:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:2
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -102,6 +122,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte2(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_bf8_byte2:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.l, v0 byte_sel:2
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -110,6 +132,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte2(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_bf8_byte2:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -123,6 +147,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte2(i32 %a) {
define amdgpu_ps float @test_cvt_f16_bf8_byte3(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_bf8_byte3:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.l, v0 byte_sel:3
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -131,6 +157,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_bf8_byte3:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:3
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -139,6 +167,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_bf8_byte3:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.l, v0 byte_sel:3
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -147,6 +177,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_bf8_byte3:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:3
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -160,6 +192,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3(i32 %a) {
define amdgpu_ps float @test_cvt_f16_bf8_byte0_s(i32 inreg %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_bf8_byte0_s:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_bf8_e32 v0.l, s0
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -168,6 +202,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0_s(i32 inreg %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_bf8_byte0_s:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_bf8_e32 v0, s0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -176,6 +212,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0_s(i32 inreg %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_bf8_byte0_s:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e32 v0.l, s0
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_3)
@@ -186,6 +224,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0_s(i32 inreg %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_bf8_byte0_s:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_bf8_e32 v0, s0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_3)
@@ -201,6 +241,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte0_s(i32 inreg %a) {
define amdgpu_ps float @test_cvt_f16_bf8_byte3_hi(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_bf8_byte3_hi:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.h, v0 byte_sel:3
; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
@@ -208,6 +250,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3_hi(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_bf8_byte3_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:3
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -216,6 +260,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3_hi(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_bf8_byte3_hi:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_bf8_e64 v0.h, v0 byte_sel:3
; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
@@ -223,6 +269,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3_hi(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_bf8_byte3_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_bf8_e64 v0, v0 byte_sel:3
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -238,6 +286,8 @@ define amdgpu_ps float @test_cvt_f16_bf8_byte3_hi(i32 %a) {
define amdgpu_ps float @test_cvt_f16_fp8_byte0(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -246,6 +296,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_fp8_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -254,6 +306,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte0:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e32 v0.l, v0
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -262,6 +316,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte0:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_fp8_e32 v0, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -275,6 +331,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0(i32 %a) {
define amdgpu_ps float @test_cvt_f16_fp8_byte1(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte1:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:1
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -283,6 +341,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte1(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte1:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:1
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -291,6 +351,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte1(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte1:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:1
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -299,6 +361,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte1(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte1:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:1
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -312,6 +376,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte1(i32 %a) {
define amdgpu_ps float @test_cvt_f16_fp8_byte2(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte2:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:2
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -320,6 +386,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte2(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte2:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:2
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -328,6 +396,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte2(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte2:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:2
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -336,6 +406,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte2(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte2:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -349,6 +421,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte2(i32 %a) {
define amdgpu_ps float @test_cvt_f16_fp8_byte3(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte3:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -357,6 +431,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte3:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -365,6 +441,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte3:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -373,6 +451,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte3:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -386,6 +466,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3(i32 %a) {
define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte3_hi:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.h, v0 byte_sel:3
; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
@@ -393,6 +475,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte3_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -401,6 +485,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte3_hi:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e64 v0.h, v0 byte_sel:3
; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
@@ -408,6 +494,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte3_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -423,6 +511,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
define amdgpu_ps float @test_cvt_f16_fp8_byte0_s(i32 inreg %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte0_s:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_f16_fp8_e32 v0.l, s0
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -431,6 +521,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0_s(i32 inreg %a) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte0_s:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_fp8_e32 v0, s0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -439,6 +531,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0_s(i32 inreg %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte0_s:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_f16_fp8_e32 v0.l, s0
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_3)
@@ -449,6 +543,8 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0_s(i32 inreg %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte0_s:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_fp8_e32 v0, s0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_3)
@@ -464,24 +560,32 @@ define amdgpu_ps float @test_cvt_f16_fp8_byte0_s(i32 inreg %a) {
define amdgpu_ps float @test_cvt_pk_f16_bf8_v(i16 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_f16_bf8_v:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_f16_bf8_v:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_f16_bf8 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_f16_bf8_v:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
; GFX1250-GISEL-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_f16_bf8_v:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_f16_bf8 v0, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
@@ -493,6 +597,8 @@ define amdgpu_ps float @test_cvt_pk_f16_bf8_v(i16 %a) {
define amdgpu_ps float @test_cvt_pk_f16_bf8_s(i16 inreg %a) {
; GFX1250-LABEL: test_cvt_pk_f16_bf8_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_f16_bf8 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -504,24 +610,32 @@ define amdgpu_ps float @test_cvt_pk_f16_bf8_s(i16 inreg %a) {
define amdgpu_ps float @test_cvt_pk_f16_fp8_v(i16 %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_f16_fp8_v:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_f16_fp8_v:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_f16_fp8 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_f16_fp8_v:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-GISEL-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_f16_fp8_v:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_f16_fp8 v0, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
@@ -533,6 +647,8 @@ define amdgpu_ps float @test_cvt_pk_f16_fp8_v(i16 %a) {
define amdgpu_ps float @test_cvt_pk_f16_fp8_s(i16 inreg %a) {
; GFX1250-LABEL: test_cvt_pk_f16_fp8_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_f16_fp8 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -544,12 +660,16 @@ define amdgpu_ps float @test_cvt_pk_f16_fp8_s(i16 inreg %a) {
define amdgpu_ps float @test_cvt_pk_f16_fp8_v_hi(<2 x i16> %a) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_f16_fp8_v_hi:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_f16_fp8 v0, v0.h
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_f16_fp8_v_hi:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -558,6 +678,8 @@ define amdgpu_ps float @test_cvt_pk_f16_fp8_v_hi(<2 x i16> %a) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_f16_fp8_v_hi:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -566,6 +688,8 @@ define amdgpu_ps float @test_cvt_pk_f16_fp8_v_hi(<2 x i16> %a) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_f16_fp8_v_hi:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll
index 241baf968491b..3c9d8ecfa7dd3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll
@@ -17,6 +17,8 @@ define amdgpu_cs float @test_cvt_f32_bf8_byte0(i32 %a) {
;
; GFX1250-LABEL: test_cvt_f32_bf8_byte0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_bf8_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-NEXT: ; return to shader part epilog
@@ -38,6 +40,8 @@ define amdgpu_cs float @test_cvt_f32_bf8_byte1(i32 %a) {
;
; GFX1250-LABEL: test_cvt_f32_bf8_byte1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_bf8_e64_dpp v0, v0 byte_sel:1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-NEXT: ; return to shader part epilog
@@ -59,6 +63,8 @@ define amdgpu_cs float @test_cvt_f32_bf8_byte2(i32 %a) {
;
; GFX1250-LABEL: test_cvt_f32_bf8_byte2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_bf8_e64_dpp v0, v0 byte_sel:2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-NEXT: ; return to shader part epilog
@@ -80,6 +86,8 @@ define amdgpu_cs float @test_cvt_f32_fp8_byte3(i32 %a) {
;
; GFX1250-LABEL: test_cvt_f32_fp8_byte3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_fp8_e64_dpp v0, v0 byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-NEXT: ; return to shader part epilog
@@ -107,6 +115,8 @@ define amdgpu_cs void @test_cvt_pk_bf8_f32_word0(i32 %a, float %y, i32 %old, ptr
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_bf8_f32_word0:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-FAKE16-NEXT: v_cvt_pk_bf8_f32_e64_dpp v2, v0, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -115,6 +125,8 @@ define amdgpu_cs void @test_cvt_pk_bf8_f32_word0(i32 %a, float %y, i32 %old, ptr
;
; GFX1250-REAL16-LABEL: test_cvt_pk_bf8_f32_word0:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -148,6 +160,8 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1(i32 %a, float %y, i32 %old, ptr
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -158,6 +172,8 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1(i32 %a, float %y, i32 %old, ptr
;
; GFX1250-REAL16-LABEL: test_cvt_pk_fp8_f32_word1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -187,6 +203,8 @@ define amdgpu_cs void @test_cvt_sr_bf8_f32_byte0(i32 %a, i32 %r, i32 %old, ptr a
;
; GFX1250-LABEL: test_cvt_sr_bf8_f32_byte0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_cvt_sr_bf8_f32_e64_dpp v2, v0, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -214,6 +232,8 @@ define amdgpu_cs void @test_cvt_sr_fp8_f32_byte1(i32 %a, i32 %r, i32 %old, ptr a
;
; GFX1250-LABEL: test_cvt_sr_fp8_f32_byte1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_cvt_sr_fp8_f32_e64_dpp v2, v0, v1 byte_sel:1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -241,6 +261,8 @@ define amdgpu_cs void @test_cvt_sr_fp8_f32_byte2(i32 %a, i32 %r, i32 %old, ptr a
;
; GFX1250-LABEL: test_cvt_sr_fp8_f32_byte2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_cvt_sr_fp8_f32_e64_dpp v2, v0, v1 byte_sel:2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
index 03e5974e2f063..2716396b2433e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll
@@ -72,6 +72,8 @@ define i32 @test_cvt_pk_fp8_f32_word1(float %x, float %y, i32 %old) {
define amdgpu_cs void @test_cvt_pk_fp8_f32_word1_dpp(i32 %a, float %y, i32 %old, ptr addrspace(1) %out) {
; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1_dpp:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -82,6 +84,8 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1_dpp(i32 %a, float %y, i32 %old,
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1_dpp:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -92,6 +96,8 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1_dpp(i32 %a, float %y, i32 %old,
;
; GFX1250-GISEL-LABEL: test_cvt_pk_fp8_f32_word1_dpp:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
@@ -161,6 +167,8 @@ define i32 @test_cvt_sr_fp8_f32_byte3(float %x, i32 %r, i32 %old) {
define amdgpu_cs void @test_cvt_sr_fp8_f32_byte1_dpp(i32 %a, i32 %r, i32 %old, ptr addrspace(1) %out) {
; GFX1250-TRUE16-LABEL: test_cvt_sr_fp8_f32_byte1_dpp:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-TRUE16-NEXT: v_cvt_sr_fp8_f32_e64_dpp v2, v0, v1 byte_sel:1 clamp quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -169,6 +177,8 @@ define amdgpu_cs void @test_cvt_sr_fp8_f32_byte1_dpp(i32 %a, i32 %r, i32 %old, p
;
; GFX1250-FAKE16-LABEL: test_cvt_sr_fp8_f32_byte1_dpp:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-FAKE16-NEXT: v_cvt_sr_fp8_f32_e64_dpp v2, v0, v1 byte_sel:1 clamp quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -177,6 +187,8 @@ define amdgpu_cs void @test_cvt_sr_fp8_f32_byte1_dpp(i32 %a, i32 %r, i32 %old, p
;
; GFX1250-GISEL-LABEL: test_cvt_sr_fp8_f32_byte1_dpp:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-NEXT: v_cvt_sr_fp8_f32_e64_dpp v2, v0, v1 byte_sel:1 clamp quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -236,6 +248,8 @@ define float @test_cvt_f32_fp8_e5m3_byte3(i32 %a) {
define amdgpu_ps i32 @test_cvt_pk_fp8_f32_e5m3_word0_sss(float inreg %x, float inreg %y, i32 inreg %old) {
; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_e5m3_word0_sss:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
@@ -246,6 +260,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_e5m3_word0_sss(float inreg %x, float i
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_e5m3_word0_sss:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -255,6 +271,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_e5m3_word0_sss(float inreg %x, float i
;
; GFX1250-GISEL-LABEL: test_cvt_pk_fp8_f32_e5m3_word0_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -268,6 +286,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_e5m3_word0_sss(float inreg %x, float i
define amdgpu_ps i32 @test_cvt_sr_fp8_f32_e5m3_byte0_sss(float inreg %x, i32 inreg %r, i32 inreg %old) {
; GFX1250-LABEL: test_cvt_sr_fp8_f32_e5m3_byte0_sss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -281,6 +301,8 @@ define amdgpu_ps i32 @test_cvt_sr_fp8_f32_e5m3_byte0_sss(float inreg %x, i32 inr
define amdgpu_ps float @test_cvt_f32_fp8_e5m3_byte0_s(i32 inreg %a) {
; GFX1250-LABEL: test_cvt_f32_fp8_e5m3_byte0_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_fp8_e64 v0, s0 clamp
; GFX1250-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
index c2aaf538da62c..155b49432801b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
@@ -12,6 +12,8 @@ declare i32 @llvm.amdgcn.cvt.sr.fp8.f16(half, i32, i32, i32)
define amdgpu_ps void @test_cvt_pk_bf8_f16_v(<2 x half> %a, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_bf8_f16_v:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_bf8_f16 v0.l, v0
@@ -20,6 +22,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_bf8_f16_v:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_bf8_f16 v0, v0
@@ -28,6 +32,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_bf8_f16_v:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_bf8_f16 v0.l, v0
@@ -36,6 +42,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_bf8_f16_v:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_bf8_f16 v0, v0
@@ -49,6 +57,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
define amdgpu_ps void @test_cvt_pk_bf8_f16_s(<2 x half> inreg %a, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_bf8_f16_s:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_bf8_f16 v2.l, s0
; GFX1250-SDAG-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -56,6 +66,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_s(<2 x half> inreg %a, ptr addrspace(
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_bf8_f16_s:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_bf8_f16 v2, s0
; GFX1250-SDAG-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -63,6 +75,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_s(<2 x half> inreg %a, ptr addrspace(
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_bf8_f16_s:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_bf8_f16 v2.l, s0
; GFX1250-GISEL-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -70,6 +84,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_s(<2 x half> inreg %a, ptr addrspace(
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_bf8_f16_s:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_bf8_f16 v2, s0
; GFX1250-GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -82,6 +98,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_s(<2 x half> inreg %a, ptr addrspace(
define amdgpu_ps void @test_cvt_pk_bf8_f16_l(ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_bf8_f16_l:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_bf8_f16 v2.l, 0x56400000
; GFX1250-SDAG-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -89,6 +107,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_l(ptr addrspace(1) %out) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_bf8_f16_l:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_bf8_f16 v2, 0x56400000
; GFX1250-SDAG-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -96,6 +116,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_l(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_bf8_f16_l:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_bf8_f16 v2.l, 0x56400000
; GFX1250-GISEL-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -103,6 +125,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_l(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_bf8_f16_l:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_bf8_f16 v2, 0x56400000
; GFX1250-GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -115,6 +139,8 @@ define amdgpu_ps void @test_cvt_pk_bf8_f16_l(ptr addrspace(1) %out) {
define amdgpu_ps void @test_cvt_pk_fp8_f16_v(<2 x half> %a, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_fp8_f16_v:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_fp8_f16 v0.l, v0
@@ -123,6 +149,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_fp8_f16_v:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_fp8_f16 v0, v0
@@ -131,6 +159,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_fp8_f16_v:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_fp8_f16 v0.l, v0
@@ -139,6 +169,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_fp8_f16_v:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_fp8_f16 v0, v0
@@ -152,6 +184,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_v(<2 x half> %a, ptr addrspace(1) %ou
define amdgpu_ps void @test_cvt_pk_fp8_f16_s(<2 x half> inreg %a, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_fp8_f16_s:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_fp8_f16 v2.l, s0
; GFX1250-SDAG-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -159,6 +193,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_s(<2 x half> inreg %a, ptr addrspace(
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_fp8_f16_s:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_fp8_f16 v2, s0
; GFX1250-SDAG-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -166,6 +202,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_s(<2 x half> inreg %a, ptr addrspace(
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_fp8_f16_s:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_fp8_f16 v2.l, s0
; GFX1250-GISEL-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -173,6 +211,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_s(<2 x half> inreg %a, ptr addrspace(
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_fp8_f16_s:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_fp8_f16 v2, s0
; GFX1250-GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -185,6 +225,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_s(<2 x half> inreg %a, ptr addrspace(
define amdgpu_ps void @test_cvt_pk_fp8_f16_l(ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_fp8_f16_l:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_cvt_pk_fp8_f16 v2.l, 0x56400000
; GFX1250-SDAG-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -192,6 +234,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_l(ptr addrspace(1) %out) {
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_fp8_f16_l:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_pk_fp8_f16 v2, 0x56400000
; GFX1250-SDAG-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -199,6 +243,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_l(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_fp8_f16_l:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_fp8_f16 v2.l, 0x56400000
; GFX1250-GISEL-REAL16-NEXT: global_store_b16 v[0:1], v2, off
@@ -206,6 +252,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_l(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_fp8_f16_l:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_pk_fp8_f16 v2, 0x56400000
; GFX1250-GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -218,6 +266,8 @@ define amdgpu_ps void @test_cvt_pk_fp8_f16_l(ptr addrspace(1) %out) {
define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_bf8_f16_byte0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1
@@ -226,6 +276,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1
@@ -234,6 +286,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_bf8_f16_byte0:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1
@@ -242,6 +296,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte0:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1
@@ -255,6 +311,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0_s(half inreg %a, i32 inreg %sr, i32 inreg %old, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_sr_bf8_f16_byte0_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -269,6 +327,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte0_s(half inreg %a, i32 inreg %sr,
define amdgpu_ps void @test_cvt_sr_bf8_f16_byte1(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_bf8_f16_byte1:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1 byte_sel:1
@@ -277,6 +337,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte1:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:1
@@ -285,6 +347,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_bf8_f16_byte1:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1 byte_sel:1
@@ -293,6 +357,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte1:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:1
@@ -306,6 +372,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_bf8_f16_byte2(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_bf8_f16_byte2:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1 byte_sel:2
@@ -314,6 +382,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte2:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:2
@@ -322,6 +392,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_bf8_f16_byte2:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1 byte_sel:2
@@ -330,6 +402,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte2:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:2
@@ -343,6 +417,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_bf8_f16_byte3(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_bf8_f16_byte3:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1 byte_sel:3
@@ -351,6 +427,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte3:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:3
@@ -359,6 +437,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_bf8_f16_byte3:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1 byte_sel:3
@@ -367,6 +447,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_bf8_f16_byte3:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1 byte_sel:3
@@ -380,6 +462,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_bf8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_bf8_f16_hi_byte0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.h, v1
@@ -388,6 +472,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_bf8_f16_hi_byte0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_lshrrev_b32 v0, 16, v0
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v4, v3
@@ -398,6 +484,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_bf8_f16_hi_byte0:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v7, v4
@@ -408,6 +496,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_bf8_f16_hi_byte0:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v7, v4
@@ -424,6 +514,8 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_fp8_f16_byte0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1
@@ -432,6 +524,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1
@@ -440,6 +534,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_fp8_f16_byte0:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1
@@ -448,6 +544,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte0:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1
@@ -461,6 +559,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0_s(half inreg %a, i32 inreg %sr, i32 inreg %old, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_sr_fp8_f16_byte0_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -475,6 +575,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte0_s(half inreg %a, i32 inreg %sr,
define amdgpu_ps void @test_cvt_sr_fp8_f16_byte1(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_fp8_f16_byte1:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1 byte_sel:1
@@ -483,6 +585,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte1:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:1
@@ -491,6 +595,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_fp8_f16_byte1:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1 byte_sel:1
@@ -499,6 +605,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte1:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:1
@@ -512,6 +620,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte1(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_fp8_f16_byte2(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_fp8_f16_byte2:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1 byte_sel:2
@@ -520,6 +630,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte2:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:2
@@ -528,6 +640,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_fp8_f16_byte2:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1 byte_sel:2
@@ -536,6 +650,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte2:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:2
@@ -549,6 +665,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte2(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_fp8_f16_byte3(half %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_fp8_f16_byte3:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1 byte_sel:3
@@ -557,6 +675,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte3:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:3
@@ -565,6 +685,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_fp8_f16_byte3:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1 byte_sel:3
@@ -573,6 +695,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_fp8_f16_byte3:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1 byte_sel:3
@@ -586,6 +710,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_byte3(half %a, i32 %sr, i32 %old, ptr
define amdgpu_ps void @test_cvt_sr_fp8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32 %old, ptr addrspace(1) %out) {
; GFX1250-SDAG-REAL16-LABEL: test_cvt_sr_fp8_f16_hi_byte0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.h, v1
@@ -594,6 +720,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
;
; GFX1250-SDAG-FAKE16-LABEL: test_cvt_sr_fp8_f16_hi_byte0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_lshrrev_b32 v0, 16, v0
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v4, v3
@@ -604,6 +732,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
;
; GFX1250-GISEL-REAL16-LABEL: test_cvt_sr_fp8_f16_hi_byte0:
; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-REAL16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v7, v4
@@ -614,6 +744,8 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_fp8_f16_hi_byte0:
; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: global_wb
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v7, v4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
index 8ad4c79233539..5c79499f11497 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll
@@ -1332,6 +1332,8 @@ define amdgpu_ps <2 x float> @test_cvt_pk_f32_bf8_word0_s(i32 inreg %a) {
;
; GFX1250-LABEL: test_cvt_pk_f32_bf8_word0_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e32 v[0:1], s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1374,6 +1376,8 @@ define amdgpu_ps <2 x float> @test_cvt_pk_f32_bf8_word1_s(i32 inreg %a) {
;
; GFX1250-LABEL: test_cvt_pk_f32_bf8_word1_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_f32_bf8_e64 v[0:1], s0 op_sel:[1,0]
; GFX1250-NEXT: ; return to shader part epilog
@@ -1399,6 +1403,8 @@ define amdgpu_ps <2 x float> @test_cvt_pk_f32_fp8_word0_s(i32 inreg %a) {
;
; GFX1250-LABEL: test_cvt_pk_f32_fp8_word0_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e32 v[0:1], s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1441,6 +1447,8 @@ define amdgpu_ps <2 x float> @test_cvt_pk_f32_fp8_word1_s(i32 inreg %a) {
;
; GFX1250-LABEL: test_cvt_pk_f32_fp8_word1_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_pk_f32_fp8_e64 v[0:1], s0 op_sel:[1,0]
; GFX1250-NEXT: ; return to shader part epilog
@@ -1477,6 +1485,8 @@ define amdgpu_ps float @test_cvt_f32_bf8_byte0_s(i32 inreg %a) {
;
; GFX1250-LABEL: test_cvt_f32_bf8_byte0_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_bf8_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1513,6 +1523,8 @@ define amdgpu_ps float @test_cvt_f32_fp8_byte0_s(i32 inreg %a) {
;
; GFX1250-LABEL: test_cvt_f32_fp8_byte0_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_f32_fp8_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1612,6 +1624,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1250-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0_sss:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
@@ -1622,6 +1636,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_bf8_f32_word0_sss:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1631,6 +1647,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1250-GISEL-LABEL: test_cvt_pk_bf8_f32_word0_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1733,6 +1751,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1250-TRUE16-LABEL: test_cvt_pk_bf8_f32_word1_sss:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
@@ -1743,6 +1763,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_bf8_f32_word1_sss:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1752,6 +1774,8 @@ define amdgpu_ps i32 @test_cvt_pk_bf8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1250-GISEL-LABEL: test_cvt_pk_bf8_f32_word1_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1854,6 +1878,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_word0_sss:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
@@ -1864,6 +1890,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word0_sss:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1873,6 +1901,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word0_sss(float inreg %x, float inreg
;
; GFX1250-GISEL-LABEL: test_cvt_pk_fp8_f32_word0_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1975,6 +2005,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1_sss:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, s2
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
@@ -1985,6 +2017,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1_sss:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1994,6 +2028,8 @@ define amdgpu_ps i32 @test_cvt_pk_fp8_f32_word1_sss(float inreg %x, float inreg
;
; GFX1250-GISEL-LABEL: test_cvt_pk_fp8_f32_word1_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2060,6 +2096,8 @@ define amdgpu_ps i32 @test_cvt_sr_bf8_f32_byte0_sss(float inreg %x, i32 inreg %r
;
; GFX1250-LABEL: test_cvt_sr_bf8_f32_byte0_sss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2126,6 +2164,8 @@ define amdgpu_ps i32 @test_cvt_sr_fp8_f32_byte0_sss(float inreg %x, i32 inreg %r
;
; GFX1250-LABEL: test_cvt_sr_fp8_f32_byte0_sss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.f16.ll
index 50936f9ff8fd4..ae23344f1605e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.f16.ll
@@ -7,6 +7,8 @@ declare <2 x half> @llvm.amdgcn.cvt.sr.pk.f16.f32(float, float, i32) #0
define amdgpu_ps float @cvt_sr_pk_f16_f32_vvv(float %src0, float %src1, i32 %src2) #1 {
; GCN-LABEL: cvt_sr_pk_f16_f32_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_f16_f32 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -18,6 +20,8 @@ define amdgpu_ps float @cvt_sr_pk_f16_f32_vvv(float %src0, float %src1, i32 %src
define amdgpu_ps float @cvt_sr_pk_f16_f32_sss(float inreg %src0, float inreg %src1, i32 inreg %src2) #1 {
; GCN-LABEL: cvt_sr_pk_f16_f32_sss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -31,6 +35,8 @@ define amdgpu_ps float @cvt_sr_pk_f16_f32_sss(float inreg %src0, float inreg %sr
define amdgpu_ps float @cvt_sr_pk_f16_f32_vvi(float %src0, float %src1) #1 {
; GCN-LABEL: cvt_sr_pk_f16_f32_vvi:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_f16_f32 v0, v0, v1, 0x10002
; GCN-NEXT: ; return to shader part epilog
@@ -42,6 +48,8 @@ define amdgpu_ps float @cvt_sr_pk_f16_f32_vvi(float %src0, float %src1) #1 {
define amdgpu_ps float @cvt_sr_pk_f16_f32_vvi_mods(float %src0, float %src1) #1 {
; GCN-LABEL: cvt_sr_pk_f16_f32_vvi_mods:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_f16_f32 v0, -v0, |v1|, 1
; GCN-NEXT: ; return to shader part epilog
@@ -55,6 +63,8 @@ define amdgpu_ps float @cvt_sr_pk_f16_f32_vvi_mods(float %src0, float %src1) #1
define amdgpu_ps float @cvt_sr_pk_f16_f32_ssi(float inreg %src0, float inreg %src1) #1 {
; GCN-LABEL: cvt_sr_pk_f16_f32_ssi:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_f16_f32 v0, s0, s1, 1
; GCN-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll
index 08dccdf5872d0..6a46a1e5dabb2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll
@@ -10,6 +10,8 @@ declare i16 @llvm.amdgcn.sat.pk4.u4.u8(i32) #0
define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
; SDAG-REAL16-LABEL: sat_pk4_i4_i8_f32_v:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_clause 0x1
; SDAG-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -22,6 +24,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
;
; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_v:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_clause 0x1
; SDAG-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -34,6 +38,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
;
; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_v:
; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_wb
+; GISEL-REAL16-NEXT: v_nop
; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-REAL16-NEXT: s_clause 0x1
; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -46,6 +52,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
;
; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_v:
; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_wb
+; GISEL-FAKE16-NEXT: v_nop
; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-FAKE16-NEXT: s_clause 0x1
; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -63,6 +71,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
; SDAG-REAL16-LABEL: sat_pk4_i4_i8_f32_s:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -73,6 +83,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
;
; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_s:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -83,6 +95,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
;
; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_s:
; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_wb
+; GISEL-REAL16-NEXT: v_nop
; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-REAL16-NEXT: s_clause 0x1
; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -95,6 +109,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
;
; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_s:
; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_wb
+; GISEL-FAKE16-NEXT: v_nop
; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-FAKE16-NEXT: s_clause 0x1
; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -112,6 +128,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
; SDAG-REAL16-LABEL: sat_pk4_i4_i8_f32_i:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -122,6 +140,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
;
; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_i:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -132,6 +152,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
;
; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_i:
; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_wb
+; GISEL-REAL16-NEXT: v_nop
; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GISEL-REAL16-NEXT: v_sat_pk4_i4_i8_e32 v0.l, 0x64
@@ -142,6 +164,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
;
; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_i:
; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_wb
+; GISEL-FAKE16-NEXT: v_nop
; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GISEL-FAKE16-NEXT: v_sat_pk4_i4_i8_e32 v0, 0x64
@@ -157,6 +181,8 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
; SDAG-REAL16-LABEL: sat_pk4_u4_u8_f32_v:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_clause 0x1
; SDAG-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -169,6 +195,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
;
; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_v:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_clause 0x1
; SDAG-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -181,6 +209,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
;
; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_v:
; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_wb
+; GISEL-REAL16-NEXT: v_nop
; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-REAL16-NEXT: s_clause 0x1
; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -193,6 +223,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
;
; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_v:
; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_wb
+; GISEL-FAKE16-NEXT: v_nop
; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-FAKE16-NEXT: s_clause 0x1
; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -210,6 +242,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
; SDAG-REAL16-LABEL: sat_pk4_u4_u8_f32_s:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -220,6 +254,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
;
; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_s:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -230,6 +266,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
;
; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_s:
; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_wb
+; GISEL-REAL16-NEXT: v_nop
; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-REAL16-NEXT: s_clause 0x1
; GISEL-REAL16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -242,6 +280,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
;
; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_s:
; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_wb
+; GISEL-FAKE16-NEXT: v_nop
; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-FAKE16-NEXT: s_clause 0x1
; GISEL-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
@@ -259,6 +299,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
; SDAG-REAL16-LABEL: sat_pk4_u4_u8_f32_i:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -269,6 +311,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
;
; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_i:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -279,6 +323,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
;
; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_i:
; GISEL-REAL16: ; %bb.0:
+; GISEL-REAL16-NEXT: global_wb
+; GISEL-REAL16-NEXT: v_nop
; GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GISEL-REAL16-NEXT: v_sat_pk4_u4_u8_e32 v0.l, 0x64
@@ -289,6 +335,8 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
;
; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_i:
; GISEL-FAKE16: ; %bb.0:
+; GISEL-FAKE16-NEXT: global_wb
+; GISEL-FAKE16-NEXT: v_nop
; GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GISEL-FAKE16-NEXT: v_sat_pk4_u4_u8_e32 v0, 0x64
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll
index 610d18ca877be..45b2a029930d3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll
@@ -21,6 +21,8 @@ declare <16 x float> @llvm.amdgcn.cvt.scale.pk16.f32.bf6(<3 x i32> %src, i32 %sc
define amdgpu_ps void @test_cvt_scale_pk8_f16_fp8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f16_fp8_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_f16_fp8 v[4:7], v[0:1], v2 scale_sel:1
@@ -29,6 +31,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_fp8_vv(<2 x i32> %src, i32 %scale,
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f16_fp8_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_f16_fp8 v[4:7], v[0:1], v2 scale_sel:1
@@ -42,6 +46,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_fp8_vv(<2 x i32> %src, i32 %scale,
define amdgpu_ps void @test_cvt_scale_pk8_f16_fp8_ss(<2 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f16_fp8_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -51,6 +57,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_fp8_ss(<2 x i32> inreg %src, i32 i
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f16_fp8_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -65,6 +73,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_fp8_ss(<2 x i32> inreg %src, i32 i
define amdgpu_ps void @test_cvt_scale_pk8_f16_bf8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f16_bf8_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_f16_bf8 v[4:7], v[0:1], v2
@@ -73,6 +83,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_bf8_vv(<2 x i32> %src, i32 %scale,
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f16_bf8_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_f16_bf8 v[4:7], v[0:1], v2
@@ -86,6 +98,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_bf8_vv(<2 x i32> %src, i32 %scale,
define amdgpu_ps void @test_cvt_scale_pk8_f16_bf8_ss(<2 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f16_bf8_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -95,6 +109,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_bf8_ss(<2 x i32> inreg %src, i32 i
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f16_bf8_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -109,6 +125,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_bf8_ss(<2 x i32> inreg %src, i32 i
define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk8_bf16_fp8_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
; GFX1250-NEXT: v_cvt_scale_pk8_bf16_fp8 v[4:7], v[0:1], v2 scale_sel:1
@@ -122,6 +140,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp8_vv(<2 x i32> %src, i32 %scale
define amdgpu_ps void @test_cvt_scale_pk8_bf16_bf8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk8_bf16_bf8_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v9, v4 :: v_dual_mov_b32 v8, v3
; GFX1250-NEXT: v_cvt_scale_pk8_bf16_bf8 v[4:7], v[0:1], v2 scale_sel:2
@@ -135,6 +155,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_bf16_bf8_vv(<2 x i32> %src, i32 %scale
define amdgpu_ps void @test_cvt_scale_pk8_f16_fp4_vv(i32 %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk8_f16_fp4_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scale_pk8_f16_fp4 v[4:7], v0, v1 scale_sel:3
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -147,6 +169,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_fp4_vv(i32 %src, i32 %scale, ptr a
define amdgpu_ps void @test_cvt_scale_pk8_f16_fp4_ss(i32 inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk8_f16_fp4_ss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -161,6 +185,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f16_fp4_ss(i32 inreg %src, i32 inreg %
define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp4_vv(i32 %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk8_bf16_fp4_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scale_pk8_bf16_fp4 v[4:7], v0, v1 scale_sel:4
; GFX1250-NEXT: global_store_b128 v[2:3], v[4:7], off
@@ -173,6 +199,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_bf16_fp4_vv(i32 %src, i32 %scale, ptr
define amdgpu_ps void @test_cvt_scale_pk8_f32_fp8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f32_fp8_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v13, v4 :: v_dual_mov_b32 v12, v3
; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_f32_fp8 v[4:11], v[0:1], v2 scale_sel:8
@@ -183,6 +211,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp8_vv(<2 x i32> %src, i32 %scale,
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f32_fp8_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v13, v4
; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_f32_fp8 v[4:11], v[0:1], v2 scale_sel:8
@@ -198,6 +228,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp8_vv(<2 x i32> %src, i32 %scale,
define amdgpu_ps void @test_cvt_scale_pk8_f32_fp8_ss(<2 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f32_fp8_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -209,6 +241,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp8_ss(<2 x i32> inreg %src, i32 i
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f32_fp8_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -240,6 +274,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp8_ss(<2 x i32> inreg %src, i32 i
define amdgpu_ps void @test_cvt_scale_pk8_f32_bf8_vv(<2 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f32_bf8_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v13, v4 :: v_dual_mov_b32 v12, v3
; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_f32_bf8 v[4:11], v[0:1], v2
@@ -250,6 +286,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_bf8_vv(<2 x i32> %src, i32 %scale,
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f32_bf8_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v13, v4
; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_f32_bf8 v[4:11], v[0:1], v2
@@ -265,6 +303,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_bf8_vv(<2 x i32> %src, i32 %scale,
define amdgpu_ps void @test_cvt_scale_pk8_f32_bf8_ss(<2 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f32_bf8_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -276,6 +316,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_bf8_ss(<2 x i32> inreg %src, i32 i
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f32_bf8_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -307,6 +349,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_bf8_ss(<2 x i32> inreg %src, i32 i
define amdgpu_ps void @test_cvt_scale_pk8_f32_fp4_vv(i32 %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f32_fp4_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scale_pk8_f32_fp4 v[4:11], v0, v1 scale_sel:1
; GFX1250-SDAG-NEXT: s_clause 0x1
@@ -316,6 +360,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp4_vv(i32 %src, i32 %scale, ptr a
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f32_fp4_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scale_pk8_f32_fp4 v[4:11], v0, v1 scale_sel:1
; GFX1250-GISEL-NEXT: s_clause 0x1
@@ -330,6 +376,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp4_vv(i32 %src, i32 %scale, ptr a
define amdgpu_ps void @test_cvt_scale_pk8_f32_fp4_ss(i32 inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk8_f32_fp4_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -341,6 +389,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp4_ss(i32 inreg %src, i32 inreg %
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk8_f32_fp4_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -372,6 +422,8 @@ define amdgpu_ps void @test_cvt_scale_pk8_f32_fp4_ss(i32 inreg %src, i32 inreg %
define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f16_fp6_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_f16_fp6 v[6:13], v[0:2], v3
; GFX1250-SDAG-NEXT: s_clause 0x1
@@ -381,6 +433,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_vv(<3 x i32> %src, i32 %scale
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f16_fp6_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_f16_fp6 v[6:13], v[0:2], v3
; GFX1250-GISEL-NEXT: s_clause 0x1
@@ -395,6 +449,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_vv(<3 x i32> %src, i32 %scale
define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_ss(<3 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f16_fp6_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v12, s2
@@ -407,6 +463,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_ss(<3 x i32> inreg %src, i32
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f16_fp6_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v10, s0
@@ -439,6 +497,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_ss(<3 x i32> inreg %src, i32
define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_sl(<3 x i32> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f16_fp6_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v12, s2
@@ -451,6 +511,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_sl(<3 x i32> inreg %src, ptr
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f16_fp6_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v10, s0
@@ -483,6 +545,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_fp6_sl(<3 x i32> inreg %src, ptr
define amdgpu_ps void @test_cvt_scale_pk16_bf16_fp6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk16_bf16_fp6_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scale_pk16_bf16_fp6 v[6:13], v[0:2], v3 scale_sel:2
; GFX1250-NEXT: s_clause 0x1
@@ -497,6 +561,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_bf16_fp6_vv(<3 x i32> %src, i32 %scal
define amdgpu_ps void @test_cvt_scale_pk16_bf16_fp6_sl(<3 x i32> inreg %src, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk16_bf16_fp6_sl:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-NEXT: v_mov_b32_e32 v12, s2
@@ -514,6 +580,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_bf16_fp6_sl(<3 x i32> inreg %src, ptr
define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f16_bf6_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_f16_bf6 v[6:13], v[0:2], v3 scale_sel:4
; GFX1250-SDAG-NEXT: s_clause 0x1
@@ -523,6 +591,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_vv(<3 x i32> %src, i32 %scale
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f16_bf6_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_f16_bf6 v[6:13], v[0:2], v3 scale_sel:4
; GFX1250-GISEL-NEXT: s_clause 0x1
@@ -537,6 +607,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_vv(<3 x i32> %src, i32 %scale
define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_ss(<3 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f16_bf6_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v12, s2
@@ -549,6 +621,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_ss(<3 x i32> inreg %src, i32
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f16_bf6_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v10, s0
@@ -581,6 +655,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_ss(<3 x i32> inreg %src, i32
define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_sl(<3 x i32> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f16_bf6_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v12, s2
@@ -593,6 +669,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_sl(<3 x i32> inreg %src, ptr
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f16_bf6_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v10, s0
@@ -625,6 +703,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f16_bf6_sl(<3 x i32> inreg %src, ptr
define amdgpu_ps void @test_cvt_scale_pk16_bf16_bf6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk16_bf16_bf6_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scale_pk16_bf16_bf6 v[6:13], v[0:2], v3 scale_sel:6
; GFX1250-NEXT: s_clause 0x1
@@ -639,6 +719,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_bf16_bf6_vv(<3 x i32> %src, i32 %scal
define amdgpu_ps void @test_cvt_scale_pk16_bf16_bf6_sl(<3 x i32> inreg %src, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_cvt_scale_pk16_bf16_bf6_sl:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v10, s0 :: v_dual_mov_b32 v11, s1
; GFX1250-NEXT: v_mov_b32_e32 v12, s2
@@ -656,6 +738,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_bf16_bf6_sl(<3 x i32> inreg %src, ptr
define amdgpu_ps void @test_cvt_scale_pk16_f32_fp6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f32_fp6_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_f32_fp6 v[6:21], v[0:2], v3 scale_sel:5
; GFX1250-SDAG-NEXT: s_clause 0x3
@@ -667,6 +751,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f32_fp6_vv(<3 x i32> %src, i32 %scale
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f32_fp6_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_f32_fp6 v[6:21], v[0:2], v3 scale_sel:5
; GFX1250-GISEL-NEXT: s_clause 0x3
@@ -683,6 +769,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f32_fp6_vv(<3 x i32> %src, i32 %scale
define amdgpu_ps void @test_cvt_scale_pk16_f32_fp6_ss(<3 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f32_fp6_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v20, s2
@@ -697,6 +785,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f32_fp6_ss(<3 x i32> inreg %src, i32
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f32_fp6_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v20, s2 :: v_dual_mov_b32 v19, s1
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v18, s0
@@ -742,6 +832,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f32_fp6_ss(<3 x i32> inreg %src, i32
define amdgpu_ps void @test_cvt_scale_pk16_f32_bf6_vv(<3 x i32> %src, i32 %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f32_bf6_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scale_pk16_f32_bf6 v[6:21], v[0:2], v3 scale_sel:6
; GFX1250-SDAG-NEXT: s_clause 0x3
@@ -753,6 +845,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f32_bf6_vv(<3 x i32> %src, i32 %scale
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f32_bf6_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scale_pk16_f32_bf6 v[6:21], v[0:2], v3 scale_sel:6
; GFX1250-GISEL-NEXT: s_clause 0x3
@@ -769,6 +863,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f32_bf6_vv(<3 x i32> %src, i32 %scale
define amdgpu_ps void @test_cvt_scale_pk16_f32_bf6_ss(<3 x i32> inreg %src, i32 inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_cvt_scale_pk16_f32_bf6_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v18, s0 :: v_dual_mov_b32 v19, s1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v20, s2
@@ -783,6 +879,8 @@ define amdgpu_ps void @test_cvt_scale_pk16_f32_bf6_ss(<3 x i32> inreg %src, i32
;
; GFX1250-GISEL-LABEL: test_cvt_scale_pk16_f32_bf6_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v20, s2 :: v_dual_mov_b32 v19, s1
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v18, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll
index 43cd6cc576db6..619125f3b20c7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll
@@ -13,6 +13,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f32_vv(<16 x float> %src, float %s
; GFX1210-SDAG-LABEL: test_scalef32_pk16_bf6_f32_vv:
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v22, v17
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk16_bf6_f32 v[18:20], v[0:15], v16
@@ -21,6 +23,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f32_vv(<16 x float> %src, float %s
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v22, v17 :: v_dual_mov_b32 v23, v18
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_bf6_f32 v[18:20], v[0:15], v16
@@ -34,6 +38,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f32_vv(<16 x float> %src, float %s
define amdgpu_ps void @test_scalef32_pk16_bf6_f32_ss(<16 x float> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -50,6 +56,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f32_ss(<16 x float> inreg %src, fl
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
@@ -71,6 +79,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f32_ss(<16 x float> inreg %src, fl
define amdgpu_ps void @test_scalef32_pk16_bf6_f32_sl(<16 x float> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -87,6 +97,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f32_sl(<16 x float> inreg %src, pt
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
@@ -108,6 +120,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f32_sl(<16 x float> inreg %src, pt
define amdgpu_ps void @test_scalef32_pk16_fp6_f32_vv(<16 x float> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v23, v18 :: v_dual_mov_b32 v22, v17
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk16_fp6_f32 v[18:20], v[0:15], v16
@@ -116,6 +130,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f32_vv(<16 x float> %src, float %s
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v22, v17 :: v_dual_mov_b32 v23, v18
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_fp6_f32 v[18:20], v[0:15], v16
@@ -129,6 +145,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f32_vv(<16 x float> %src, float %s
define amdgpu_ps void @test_scalef32_pk16_fp6_f32_ss(<16 x float> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -145,6 +163,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f32_ss(<16 x float> inreg %src, fl
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
@@ -166,6 +186,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f32_ss(<16 x float> inreg %src, fl
define amdgpu_ps void @test_scalef32_pk16_fp6_f32_sl(<16 x float> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -182,6 +204,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f32_sl(<16 x float> inreg %src, pt
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
@@ -203,6 +227,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f32_sl(<16 x float> inreg %src, pt
define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_vv(<16 x bfloat> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk16_bf6_bf16 v[10:12], v[0:7], v8
@@ -211,6 +237,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_vv(<16 x bfloat> %src, float
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_bf6_bf16 v[10:12], v[0:7], v8
@@ -224,6 +252,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_vv(<16 x bfloat> %src, float
define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_sl(<16 x bfloat> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -236,6 +266,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_sl(<16 x bfloat> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -253,6 +285,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_bf16_sl(<16 x bfloat> inreg %src,
define amdgpu_ps void @test_scalef32_pk16_bf6_f16_vv(<16 x half> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk16_bf6_f16 v[10:12], v[0:7], v8
@@ -261,6 +295,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f16_vv(<16 x half> %src, float %sc
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v14, v9 :: v_dual_mov_b32 v15, v10
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_bf6_f16 v[10:12], v[0:7], v8
@@ -274,6 +310,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f16_vv(<16 x half> %src, float %sc
define amdgpu_ps void @test_scalef32_pk16_bf6_f16_ss(<16 x half> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -286,6 +324,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f16_ss(<16 x half> inreg %src, flo
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -303,6 +343,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f16_ss(<16 x half> inreg %src, flo
define amdgpu_ps void @test_scalef32_pk16_bf6_f16_sl(<16 x half> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_bf6_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -315,6 +357,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f16_sl(<16 x half> inreg %src, ptr
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_bf6_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -332,6 +376,8 @@ define amdgpu_ps void @test_scalef32_pk16_bf6_f16_sl(<16 x half> inreg %src, ptr
define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_vv(<16 x bfloat> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk16_fp6_bf16 v[10:12], v[0:7], v8
@@ -340,6 +386,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_vv(<16 x bfloat> %src, float
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_fp6_bf16 v[10:12], v[0:7], v8
@@ -353,6 +401,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_vv(<16 x bfloat> %src, float
define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_sl(<16 x bfloat> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -365,6 +415,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_sl(<16 x bfloat> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -382,6 +434,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_bf16_sl(<16 x bfloat> inreg %src,
define amdgpu_ps void @test_scalef32_pk16_fp6_f16_vv(<16 x half> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v15, v10 :: v_dual_mov_b32 v14, v9
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk16_fp6_f16 v[10:12], v[0:7], v8
@@ -390,6 +444,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f16_vv(<16 x half> %src, float %sc
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v14, v9 :: v_dual_mov_b32 v15, v10
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk16_fp6_f16 v[10:12], v[0:7], v8
@@ -403,6 +459,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f16_vv(<16 x half> %src, float %sc
define amdgpu_ps void @test_scalef32_pk16_fp6_f16_ss(<16 x half> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -415,6 +473,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f16_ss(<16 x half> inreg %src, flo
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -432,6 +492,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f16_ss(<16 x half> inreg %src, flo
define amdgpu_ps void @test_scalef32_pk16_fp6_f16_sl(<16 x half> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk16_fp6_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -444,6 +506,8 @@ define amdgpu_ps void @test_scalef32_pk16_fp6_f16_sl(<16 x half> inreg %src, ptr
;
; GFX1250-GISEL-LABEL: test_scalef32_pk16_fp6_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll
index 00fb6ea7cc313..d6a26c8142c96 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll
@@ -15,6 +15,8 @@ declare i32 @llvm.amdgcn.cvt.scalef32.pk8.fp4.bf16(<8 x bfloat> %src, float %sca
define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_vv(<8 x bfloat> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_fp8_bf16 v[8:9], v[0:3], v4
@@ -23,6 +25,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_vv(<8 x bfloat> %src, float %s
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp8_bf16 v[8:9], v[0:3], v4
@@ -36,6 +40,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_vv(<8 x bfloat> %src, float %s
define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -46,6 +52,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src, pt
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -61,6 +69,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src, pt
define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_vv(<8 x bfloat> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_bf8_bf16 v[8:9], v[0:3], v4
@@ -69,6 +79,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_vv(<8 x bfloat> %src, float %s
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_bf8_bf16 v[8:9], v[0:3], v4
@@ -82,6 +94,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_vv(<8 x bfloat> %src, float %s
define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -92,6 +106,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src, pt
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -107,6 +123,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src, pt
define amdgpu_ps void @test_scalef32_pk8_fp8_f16_vv(<8 x half> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_fp8_f16 v[8:9], v[0:3], v4
@@ -115,6 +133,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f16_vv(<8 x half> %src, float %scal
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v9, v6
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp8_f16 v[6:7], v[0:3], v4
@@ -128,6 +148,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f16_vv(<8 x half> %src, float %scal
define amdgpu_ps void @test_scalef32_pk8_fp8_f16_ss(<8 x half> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -138,6 +160,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f16_ss(<8 x half> inreg %src, float
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -153,6 +177,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f16_ss(<8 x half> inreg %src, float
define amdgpu_ps void @test_scalef32_pk8_fp8_f16_sl(<8 x half> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -163,6 +189,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f16_sl(<8 x half> inreg %src, ptr a
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -178,6 +206,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f16_sl(<8 x half> inreg %src, ptr a
define amdgpu_ps void @test_scalef32_pk8_bf8_f16_vv(<8 x half> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_bf8_f16 v[8:9], v[0:3], v4
@@ -186,6 +216,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f16_vv(<8 x half> %src, float %scal
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v9, v6
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_bf8_f16 v[6:7], v[0:3], v4
@@ -199,6 +231,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f16_vv(<8 x half> %src, float %scal
define amdgpu_ps void @test_scalef32_pk8_bf8_f16_ss(<8 x half> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -209,6 +243,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f16_ss(<8 x half> inreg %src, float
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -224,6 +260,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f16_ss(<8 x half> inreg %src, float
define amdgpu_ps void @test_scalef32_pk8_bf8_f16_sl(<8 x half> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -234,6 +272,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f16_sl(<8 x half> inreg %src, ptr a
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -249,6 +289,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f16_sl(<8 x half> inreg %src, ptr a
define amdgpu_ps void @test_scalef32_pk8_bf8_f32_vv(<8 x float> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v10, v9
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_bf8_f32 v[12:13], v[0:7], v8
@@ -257,6 +299,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f32_vv(<8 x float> %src, float %sca
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v10
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_bf8_f32 v[10:11], v[0:7], v8
@@ -270,6 +314,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f32_vv(<8 x float> %src, float %sca
define amdgpu_ps void @test_scalef32_pk8_bf8_f32_ss(<8 x float> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -282,6 +328,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f32_ss(<8 x float> inreg %src, floa
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -299,6 +347,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f32_ss(<8 x float> inreg %src, floa
define amdgpu_ps void @test_scalef32_pk8_bf8_f32_sl(<8 x float> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_bf8_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -311,6 +361,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f32_sl(<8 x float> inreg %src, ptr
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_bf8_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -328,6 +380,8 @@ define amdgpu_ps void @test_scalef32_pk8_bf8_f32_sl(<8 x float> inreg %src, ptr
define amdgpu_ps void @test_scalef32_pk8_fp8_f32_vv(<8 x float> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v10, v9
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_fp8_f32 v[12:13], v[0:7], v8
@@ -336,6 +390,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f32_vv(<8 x float> %src, float %sca
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v10
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp8_f32 v[10:11], v[0:7], v8
@@ -349,6 +405,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f32_vv(<8 x float> %src, float %sca
define amdgpu_ps void @test_scalef32_pk8_fp8_f32_ss(<8 x float> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -361,6 +419,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f32_ss(<8 x float> inreg %src, floa
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -378,6 +438,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f32_ss(<8 x float> inreg %src, floa
define amdgpu_ps void @test_scalef32_pk8_fp8_f32_sl(<8 x float> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp8_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -390,6 +452,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f32_sl(<8 x float> inreg %src, ptr
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp8_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -407,6 +471,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp8_f32_sl(<8 x float> inreg %src, ptr
define amdgpu_ps void @test_scalef32_pk8_fp4_f32_vv(<8 x float> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v10, v9
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_fp4_f32 v9, v[0:7], v8
@@ -415,6 +481,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f32_vv(<8 x float> %src, float %sca
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v12, v9 :: v_dual_mov_b32 v13, v10
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp4_f32 v9, v[0:7], v8
@@ -428,6 +496,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f32_vv(<8 x float> %src, float %sca
define amdgpu_ps void @test_scalef32_pk8_fp4_f32_ss(<8 x float> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -440,6 +510,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f32_ss(<8 x float> inreg %src, floa
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -457,6 +529,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f32_ss(<8 x float> inreg %src, floa
define amdgpu_ps void @test_scalef32_pk8_fp4_f32_sl(<8 x float> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -469,6 +543,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f32_sl(<8 x float> inreg %src, ptr
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -486,6 +562,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f32_sl(<8 x float> inreg %src, ptr
define amdgpu_ps void @test_scalef32_pk8_fp4_f16_vv(<8 x half> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_fp4_f16 v5, v[0:3], v4
@@ -494,6 +572,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f16_vv(<8 x half> %src, float %scal
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v9, v6
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp4_f16 v5, v[0:3], v4
@@ -507,6 +587,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f16_vv(<8 x half> %src, float %scal
define amdgpu_ps void @test_scalef32_pk8_fp4_f16_ss(<8 x half> inreg %src, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -517,6 +599,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f16_ss(<8 x half> inreg %src, float
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -532,6 +616,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f16_ss(<8 x half> inreg %src, float
define amdgpu_ps void @test_scalef32_pk8_fp4_f16_sl(<8 x half> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -542,6 +628,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f16_sl(<8 x half> inreg %src, ptr a
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -557,6 +645,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_f16_sl(<8 x half> inreg %src, ptr a
define amdgpu_ps void @test_scalef32_pk8_fp4_bf16_vv(<8 x bfloat> %src, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-SDAG-NEXT: v_cvt_scalef32_pk8_fp4_bf16 v5, v[0:3], v4
@@ -565,6 +655,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_bf16_vv(<8 x bfloat> %src, float %s
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-GISEL-NEXT: v_cvt_scalef32_pk8_fp4_bf16 v5, v[0:3], v4
@@ -578,6 +670,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_bf16_vv(<8 x bfloat> %src, float %s
define amdgpu_ps void @test_scalef32_pk8_fp4_bf16_sl(<8 x bfloat> inreg %src, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_pk8_fp4_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -588,6 +682,8 @@ define amdgpu_ps void @test_scalef32_pk8_fp4_bf16_sl(<8 x bfloat> inreg %src, pt
;
; GFX1250-GISEL-LABEL: test_scalef32_pk8_fp4_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll
index 803281b9ac9d4..fe859c4bde8b9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll
@@ -15,6 +15,8 @@ declare i32 @llvm.amdgcn.cvt.scalef32.sr.pk8.fp4.bf16(<8 x bfloat> %src, i32 %sr
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_bf16_vv(<8 x bfloat> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_fp8_bf16 v[8:9], v[0:3], v4, v5
; GFX1250-SDAG-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -22,6 +24,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_bf16_vv(<8 x bfloat> %src, i32 %
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp8_bf16 v[8:9], v[0:3], v4, v5
; GFX1250-GISEL-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -34,6 +38,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_bf16_vv(<8 x bfloat> %src, i32 %
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -44,6 +50,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -59,6 +67,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_bf16_sl(<8 x bfloat> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_bf16_vv(<8 x bfloat> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_bf8_bf16 v[8:9], v[0:3], v4, v5
; GFX1250-SDAG-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -66,6 +76,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_bf16_vv(<8 x bfloat> %src, i32 %
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_bf8_bf16 v[8:9], v[0:3], v4, v5
; GFX1250-GISEL-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -78,6 +90,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_bf16_vv(<8 x bfloat> %src, i32 %
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -88,6 +102,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -103,6 +119,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_bf16_sl(<8 x bfloat> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_vv(<8 x half> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_fp8_f16 v[8:9], v[0:3], v4, v5
; GFX1250-SDAG-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -110,6 +128,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_vv(<8 x half> %src, i32 %sr,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp8_f16 v[8:9], v[0:3], v4, v5
; GFX1250-GISEL-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -122,6 +142,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_vv(<8 x half> %src, i32 %sr,
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_ss(<8 x half> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -132,6 +154,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_ss(<8 x half> inreg %src, i3
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -147,6 +171,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_ss(<8 x half> inreg %src, i3
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_sl(<8 x half> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -157,6 +183,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_sl(<8 x half> inreg %src, i3
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -172,6 +200,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f16_sl(<8 x half> inreg %src, i3
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_vv(<8 x half> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_bf8_f16 v[8:9], v[0:3], v4, v5
; GFX1250-SDAG-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -179,6 +209,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_vv(<8 x half> %src, i32 %sr,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_bf8_f16 v[8:9], v[0:3], v4, v5
; GFX1250-GISEL-NEXT: global_store_b64 v[6:7], v[8:9], off
@@ -191,6 +223,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_vv(<8 x half> %src, i32 %sr,
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_ss(<8 x half> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -201,6 +235,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_ss(<8 x half> inreg %src, i3
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -216,6 +252,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_ss(<8 x half> inreg %src, i3
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_sl(<8 x half> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -226,6 +264,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_sl(<8 x half> inreg %src, i3
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -241,6 +281,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f16_sl(<8 x half> inreg %src, i3
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_vv(<8 x float> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_bf8_f32 v[12:13], v[0:7], v8, v9
; GFX1250-SDAG-NEXT: global_store_b64 v[10:11], v[12:13], off
@@ -248,6 +290,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_vv(<8 x float> %src, i32 %sr
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_bf8_f32 v[12:13], v[0:7], v8, v9
; GFX1250-GISEL-NEXT: global_store_b64 v[10:11], v[12:13], off
@@ -260,6 +304,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_vv(<8 x float> %src, i32 %sr
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_ss(<8 x float> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -272,6 +318,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_ss(<8 x float> inreg %src, i
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -289,6 +337,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_ss(<8 x float> inreg %src, i
define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_sl(<8 x float> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_bf8_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -301,6 +351,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_sl(<8 x float> inreg %src, i
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_bf8_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -318,6 +370,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_bf8_f32_sl(<8 x float> inreg %src, i
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_vv(<8 x float> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_fp8_f32 v[12:13], v[0:7], v8, v9
; GFX1250-SDAG-NEXT: global_store_b64 v[10:11], v[12:13], off
@@ -325,6 +379,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_vv(<8 x float> %src, i32 %sr
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp8_f32 v[12:13], v[0:7], v8, v9
; GFX1250-GISEL-NEXT: global_store_b64 v[10:11], v[12:13], off
@@ -337,6 +393,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_vv(<8 x float> %src, i32 %sr
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_ss(<8 x float> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -349,6 +407,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_ss(<8 x float> inreg %src, i
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -366,6 +426,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_ss(<8 x float> inreg %src, i
define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_sl(<8 x float> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp8_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -378,6 +440,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_sl(<8 x float> inreg %src, i
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp8_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -395,6 +459,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp8_f32_sl(<8 x float> inreg %src, i
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_vv(<8 x float> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_f32_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_fp4_f32 v12, v[0:7], v8, v9
; GFX1250-SDAG-NEXT: global_store_b32 v[10:11], v12, off
@@ -402,6 +468,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_vv(<8 x float> %src, i32 %sr
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_f32_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp4_f32 v12, v[0:7], v8, v9
; GFX1250-GISEL-NEXT: global_store_b32 v[10:11], v12, off
@@ -414,6 +482,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_vv(<8 x float> %src, i32 %sr
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_ss(<8 x float> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -426,6 +496,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_ss(<8 x float> inreg %src, i
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -443,6 +515,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_ss(<8 x float> inreg %src, i
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_sl(<8 x float> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -455,6 +529,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_sl(<8 x float> inreg %src, i
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -472,6 +548,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f32_sl(<8 x float> inreg %src, i
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_vv(<8 x half> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_f16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_fp4_f16 v8, v[0:3], v4, v5
; GFX1250-SDAG-NEXT: global_store_b32 v[6:7], v8, off
@@ -479,6 +557,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_vv(<8 x half> %src, i32 %sr,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_f16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp4_f16 v8, v[0:3], v4, v5
; GFX1250-GISEL-NEXT: global_store_b32 v[6:7], v8, off
@@ -491,6 +571,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_vv(<8 x half> %src, i32 %sr,
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_ss(<8 x half> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -501,6 +583,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_ss(<8 x half> inreg %src, i3
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -516,6 +600,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_ss(<8 x half> inreg %src, i3
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_sl(<8 x half> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -526,6 +612,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_sl(<8 x half> inreg %src, i3
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
@@ -541,6 +629,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_f16_sl(<8 x half> inreg %src, i3
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_bf16_vv(<8 x bfloat> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_bf16_vv:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_cvt_scalef32_sr_pk8_fp4_bf16 v8, v[0:3], v4, v5
; GFX1250-SDAG-NEXT: global_store_b32 v[6:7], v8, off
@@ -548,6 +638,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_bf16_vv(<8 x bfloat> %src, i32 %
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_bf16_vv:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_cvt_scalef32_sr_pk8_fp4_bf16 v8, v[0:3], v4, v5
; GFX1250-GISEL-NEXT: global_store_b32 v[6:7], v8, off
@@ -560,6 +652,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_bf16_vv(<8 x bfloat> %src, i32 %
define amdgpu_ps void @test_scalef32_sr_pk8_fp4_bf16_sl(<8 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk8_fp4_bf16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -570,6 +664,8 @@ define amdgpu_ps void @test_scalef32_sr_pk8_fp4_bf16_sl(<8 x bfloat> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk8_fp4_bf16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll
index 96a019207fd51..a095050741057 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll
@@ -12,6 +12,8 @@ declare <3 x i32> @llvm.amdgcn.cvt.scalef32.sr.pk16.fp6.f32(<16 x float> %src, i
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_bf16_vv(<16 x bfloat> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_bf6_bf16_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_bf6_bf16 v[12:14], v[0:7], v8, v9
; GFX1250-NEXT: global_store_b96 v[10:11], v[12:14], off
@@ -24,6 +26,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_bf16_vv(<16 x bfloat> %src, i32
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_bf16_sl(<16 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_bf6_bf16_sl:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -41,6 +45,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_bf16_sl(<16 x bfloat> inreg %sr
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_vv(<16 x half> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_bf6_f16_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_bf6_f16 v[12:14], v[0:7], v8, v9
; GFX1250-NEXT: global_store_b96 v[10:11], v[12:14], off
@@ -53,6 +59,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_vv(<16 x half> %src, i32 %s
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_ss(<16 x half> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_bf6_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -65,6 +73,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_ss(<16 x half> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_bf6_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -82,6 +92,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_ss(<16 x half> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_sl(<16 x half> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_bf6_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -94,6 +106,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_sl(<16 x half> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_bf6_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -111,6 +125,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f16_sl(<16 x half> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_bf16_vv(<16 x bfloat> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_fp6_bf16_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_fp6_bf16 v[12:14], v[0:7], v8, v9
; GFX1250-NEXT: global_store_b96 v[10:11], v[12:14], off
@@ -123,6 +139,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_bf16_vv(<16 x bfloat> %src, i32
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_bf16_sl(<16 x bfloat> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_fp6_bf16_sl:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -140,6 +158,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_bf16_sl(<16 x bfloat> inreg %sr
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_vv(<16 x half> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_fp6_f16_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_fp6_f16 v[12:14], v[0:7], v8, v9
; GFX1250-NEXT: global_store_b96 v[10:11], v[12:14], off
@@ -152,6 +172,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_vv(<16 x half> %src, i32 %s
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_ss(<16 x half> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_fp6_f16_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -164,6 +186,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_ss(<16 x half> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_fp6_f16_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -181,6 +205,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_ss(<16 x half> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_sl(<16 x half> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_fp6_f16_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -193,6 +219,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_sl(<16 x half> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_fp6_f16_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5]
@@ -210,6 +238,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f16_sl(<16 x half> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_vv(<16 x float> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_bf6_f32_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_bf6_f32 v[20:22], v[0:15], v16, v17
; GFX1250-NEXT: global_store_b96 v[18:19], v[20:22], off
@@ -222,6 +252,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_vv(<16 x float> %src, i32 %
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_ss(<16 x float> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_bf6_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -238,6 +270,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_ss(<16 x float> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_bf6_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
@@ -259,6 +293,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_ss(<16 x float> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_sl(<16 x float> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_bf6_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -275,6 +311,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_sl(<16 x float> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_bf6_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
@@ -296,6 +334,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_bf6_f32_sl(<16 x float> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f32_vv(<16 x float> %src, i32 %sr, float %scale, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_scalef32_sr_pk16_fp6_f32_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_scalef32_sr_pk16_fp6_f32 v[20:22], v[0:15], v16, v17
; GFX1250-NEXT: global_store_b96 v[18:19], v[20:22], off
@@ -308,6 +348,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f32_vv(<16 x float> %src, i32 %
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f32_ss(<16 x float> inreg %src, i32 inreg %sr, float inreg %scale, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_fp6_f32_ss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -324,6 +366,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f32_ss(<16 x float> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_fp6_f32_ss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
@@ -345,6 +389,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f32_ss(<16 x float> inreg %src,
define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f32_sl(<16 x float> inreg %src, i32 inreg %sr, ptr addrspace(1) %out) {
; GFX1250-SDAG-LABEL: test_scalef32_sr_pk16_fp6_f32_sl:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
@@ -361,6 +407,8 @@ define amdgpu_ps void @test_scalef32_sr_pk16_fp6_f32_sl(<16 x float> inreg %src,
;
; GFX1250-GISEL-LABEL: test_scalef32_sr_pk16_fp6_f32_sl:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[14:15]
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[12:13]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll
index 2dccb991bd14c..857d4fd944f1f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll
@@ -9,6 +9,8 @@ declare <2 x bfloat> @llvm.amdgcn.cvt.sr.pk.bf16.f32(float, float, i32) #0
define amdgpu_ps float @cvt_sr_pk_bf16_f32_vvv(float %src0, float %src1, i32 %src2) #1 {
; GCN-LABEL: cvt_sr_pk_bf16_f32_vvv:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_bf16_f32 v0, v0, v1, v2
; GCN-NEXT: ; return to shader part epilog
@@ -20,6 +22,8 @@ define amdgpu_ps float @cvt_sr_pk_bf16_f32_vvv(float %src0, float %src1, i32 %sr
define amdgpu_ps float @cvt_sr_pk_bf16_f32_sss(float inreg %src0, float inreg %src1, i32 inreg %src2) #1 {
; GCN-LABEL: cvt_sr_pk_bf16_f32_sss:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -33,6 +37,8 @@ define amdgpu_ps float @cvt_sr_pk_bf16_f32_sss(float inreg %src0, float inreg %s
define amdgpu_ps float @cvt_sr_pk_bf16_f32_vvi(float %src0, float %src1) #1 {
; GCN-LABEL: cvt_sr_pk_bf16_f32_vvi:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_bf16_f32 v0, v0, v1, 0x10002
; GCN-NEXT: ; return to shader part epilog
@@ -44,6 +50,8 @@ define amdgpu_ps float @cvt_sr_pk_bf16_f32_vvi(float %src0, float %src1) #1 {
define amdgpu_ps float @cvt_sr_pk_bf16_f32_vvi_mods(float %src0, float %src1) #1 {
; GCN-LABEL: cvt_sr_pk_bf16_f32_vvi_mods:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_bf16_f32 v0, -v0, |v1|, 1
; GCN-NEXT: ; return to shader part epilog
@@ -57,6 +65,8 @@ define amdgpu_ps float @cvt_sr_pk_bf16_f32_vvi_mods(float %src0, float %src1) #1
define amdgpu_ps float @cvt_sr_pk_bf16_f32_ssi(float inreg %src0, float inreg %src1) #1 {
; GCN-LABEL: cvt_sr_pk_bf16_f32_ssi:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_cvt_sr_pk_bf16_f32 v0, s0, s1, 1
; GCN-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
index f7211cb8a3a36..13dea3d2fc978 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
@@ -36,6 +36,8 @@ entry:
define amdgpu_ps void @test_ds_atomic_async_barrier_arrive_b64_s(ptr addrspace(3) inreg %bar) {
; GCN-LABEL: test_ds_atomic_async_barrier_arrive_b64_s:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: s_wait_alu depctr_vm_vsrc(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
index 3bcf345561884..ded5a15ee0a47 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
@@ -66,6 +66,8 @@ entry:
define amdgpu_ps void @test_ds_atomic_barrier_arrive_rtn_b64_ss(ptr addrspace(3) inreg %bar, i64 inreg %data, ptr inreg %out) {
; SDAG-LABEL: test_ds_atomic_barrier_arrive_rtn_b64_ss:
; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s2
; SDAG-NEXT: v_mov_b32_e32 v2, s0
@@ -79,6 +81,8 @@ define amdgpu_ps void @test_ds_atomic_barrier_arrive_rtn_b64_ss(ptr addrspace(3)
;
; GISEL-LABEL: test_ds_atomic_barrier_arrive_rtn_b64_ss:
; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s6, s1
; GISEL-NEXT: s_mov_b32 s7, s2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
index a85676a7ac7df..3e9e7d6d090dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.amdgcn.exp2.bf16(bfloat) #0
define amdgpu_kernel void @exp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-LABEL: exp_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -21,6 +23,8 @@ define amdgpu_kernel void @exp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; REAL16-LABEL: exp_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -36,6 +40,8 @@ define amdgpu_kernel void @exp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @exp_bf16_constant_4(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: exp_bf16_constant_4:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_exp_bf16_e32 v0, 4.0
@@ -46,6 +52,8 @@ define amdgpu_kernel void @exp_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: exp_bf16_constant_4:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_exp_bf16_e32 v0.l, 4.0
@@ -61,6 +69,8 @@ define amdgpu_kernel void @exp_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @exp_bf16_constant_100(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: exp_bf16_constant_100:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_exp_bf16_e32 v0, 0x42c8
@@ -71,6 +81,8 @@ define amdgpu_kernel void @exp_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: exp_bf16_constant_100:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_exp_bf16_e32 v0.l, 0x42c8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
index c7e698a358bd6..ce25ef52e50eb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
@@ -7,6 +7,8 @@ declare void @llvm.amdgcn.flat.prefetch(ptr %ptr, i32 %col)
define amdgpu_ps void @flat_prefetch(ptr %ptr) {
; GCN-LABEL: flat_prefetch:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v[0:1]
; GCN-NEXT: s_endpgm
@@ -18,6 +20,8 @@ entry:
define amdgpu_ps void @flat_prefetch_sgpr(ptr inreg %ptr) {
; GCN-LABEL: flat_prefetch_sgpr:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: flat_prefetch_b8 v0, s[0:1]
@@ -30,6 +34,8 @@ entry:
define amdgpu_ps void @flat_prefetch_offset(ptr %ptr) {
; GCN-LABEL: flat_prefetch_offset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v[0:1] offset:512
; GCN-NEXT: s_endpgm
@@ -42,6 +48,8 @@ entry:
define amdgpu_ps void @flat_prefetch_sgpr_voffset(ptr inreg %ptr, i32 %offset) {
; GCN-LABEL: flat_prefetch_sgpr_voffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v0, s[0:1]
; GCN-NEXT: s_endpgm
@@ -54,6 +62,8 @@ entry:
define amdgpu_ps void @flat_prefetch_sgpr_voffset_offset(ptr inreg %ptr, i32 %offset) {
; GCN-LABEL: flat_prefetch_sgpr_voffset_offset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v0, s[0:1] offset:128
; GCN-NEXT: s_endpgm
@@ -67,6 +77,8 @@ entry:
define amdgpu_ps void @flat_prefetch_se(ptr %ptr) {
; GCN-LABEL: flat_prefetch_se:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_SE
; GCN-NEXT: s_endpgm
@@ -78,6 +90,8 @@ entry:
define amdgpu_ps void @flat_prefetch_se_nt(ptr %ptr) {
; GCN-LABEL: flat_prefetch_se_nt:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v[0:1] th:TH_LOAD_NT scope:SCOPE_SE
; GCN-NEXT: s_endpgm
@@ -89,6 +103,8 @@ entry:
define amdgpu_ps void @flat_prefetch_dev_ht(ptr %ptr) {
; GCN-LABEL: flat_prefetch_dev_ht:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v[0:1] th:TH_LOAD_HT scope:SCOPE_DEV
; GCN-NEXT: s_endpgm
@@ -100,6 +116,8 @@ entry:
define amdgpu_ps void @flat_prefetch_sys_lu(ptr %ptr) {
; GCN-LABEL: flat_prefetch_sys_lu:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_prefetch_b8 v[0:1] th:TH_LOAD_BYPASS scope:SCOPE_SYS
; GCN-NEXT: s_endpgm
@@ -111,6 +129,8 @@ entry:
define amdgpu_ps float @flat_prefetch_and_load_b32_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_prefetch_and_load_b32_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
index ed46f3753ceb8..e3393acd9df4a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll
@@ -10,6 +10,8 @@ declare void @llvm.amdgcn.global.load.async.to.lds.b128(ptr addrspace(1) %gaddr,
define amdgpu_ps void @global_load_async_to_lds_b8_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_load_async_to_lds_b8_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b8 v2, v[0:1], off offset:16 th:TH_LOAD_NT
@@ -17,6 +19,8 @@ define amdgpu_ps void @global_load_async_to_lds_b8_vaddr(ptr addrspace(1) %gaddr
;
; GFX1250-GISEL-LABEL: global_load_async_to_lds_b8_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -32,6 +36,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b8_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_load_async_to_lds_b8_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_load_async_to_lds_b8 v0, v1, s[0:1] offset:16
@@ -45,6 +51,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b32_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_load_async_to_lds_b32_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b32 v2, v[0:1], off offset:16 th:TH_LOAD_HT scope:SCOPE_SE
@@ -52,6 +60,8 @@ define amdgpu_ps void @global_load_async_to_lds_b32_vaddr(ptr addrspace(1) %gadd
;
; GFX1250-GISEL-LABEL: global_load_async_to_lds_b32_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -67,6 +77,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b32_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_load_async_to_lds_b32_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:16
@@ -80,6 +92,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b64_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_load_async_to_lds_b64_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b64 v2, v[0:1], off offset:16 th:TH_LOAD_NT_HT scope:SCOPE_DEV
@@ -87,6 +101,8 @@ define amdgpu_ps void @global_load_async_to_lds_b64_vaddr(ptr addrspace(1) %gadd
;
; GFX1250-GISEL-LABEL: global_load_async_to_lds_b64_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -102,6 +118,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b64_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_load_async_to_lds_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_load_async_to_lds_b64 v0, v1, s[0:1] offset:16
@@ -115,6 +133,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b128_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_load_async_to_lds_b128_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_load_async_to_lds_b128 v2, v[0:1], off offset:16 th:TH_LOAD_BYPASS scope:SCOPE_SYS
@@ -122,6 +142,8 @@ define amdgpu_ps void @global_load_async_to_lds_b128_vaddr(ptr addrspace(1) %gad
;
; GFX1250-GISEL-LABEL: global_load_async_to_lds_b128_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -137,6 +159,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b128_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_load_async_to_lds_b128_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v1, s[0:1] offset:16
@@ -150,6 +174,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b32_saddr_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %idx) {
; GFX1250-LABEL: global_load_async_to_lds_b32_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_async_to_lds_b32 v0, v1, s[0:1] offset:16 scale_offset th:TH_LOAD_NT
; GFX1250-NEXT: s_endpgm
@@ -163,6 +189,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b64_saddr_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %idx) {
; GFX1250-LABEL: global_load_async_to_lds_b64_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_async_to_lds_b64 v0, v1, s[0:1] offset:16 scale_offset th:TH_LOAD_NT
; GFX1250-NEXT: s_endpgm
@@ -176,6 +204,8 @@ entry:
define amdgpu_ps void @global_load_async_to_lds_b64_saddr_no_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %idx) {
; GFX1250-LABEL: global_load_async_to_lds_b64_saddr_no_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
index 035f2e62b29f4..5f2855ad129dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
@@ -7,6 +7,8 @@ declare void @llvm.amdgcn.global.prefetch(ptr addrspace(1) %ptr, i32 %col)
define amdgpu_ps void @global_prefetch(ptr addrspace(1) %ptr) {
; GCN-LABEL: global_prefetch:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v[0:1], off
; GCN-NEXT: s_endpgm
@@ -18,6 +20,8 @@ entry:
define amdgpu_ps void @global_prefetch_sgpr(ptr addrspace(1) inreg %ptr) {
; GCN-LABEL: global_prefetch_sgpr:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: global_prefetch_b8 v0, s[0:1]
@@ -30,6 +34,8 @@ entry:
define amdgpu_ps void @global_prefetch_offset(ptr addrspace(1) %ptr) {
; GCN-LABEL: global_prefetch_offset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v[0:1], off offset:512
; GCN-NEXT: s_endpgm
@@ -42,6 +48,8 @@ entry:
define amdgpu_ps void @global_prefetch_sgpr_voffset(ptr addrspace(1) inreg %ptr, i32 %offset) {
; GCN-LABEL: global_prefetch_sgpr_voffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v0, s[0:1]
; GCN-NEXT: s_endpgm
@@ -54,6 +62,8 @@ entry:
define amdgpu_ps void @global_prefetch_sgpr_voffset_offset(ptr addrspace(1) inreg %ptr, i32 %offset) {
; GCN-LABEL: global_prefetch_sgpr_voffset_offset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v0, s[0:1] offset:128
; GCN-NEXT: s_endpgm
@@ -67,6 +77,8 @@ entry:
define amdgpu_ps void @global_prefetch_se(ptr addrspace(1) %ptr) {
; GCN-LABEL: global_prefetch_se:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v[0:1], off scope:SCOPE_SE
; GCN-NEXT: s_endpgm
@@ -78,6 +90,8 @@ entry:
define amdgpu_ps void @global_prefetch_se_nt(ptr addrspace(1) %ptr) {
; GCN-LABEL: global_prefetch_se_nt:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v[0:1], off th:TH_LOAD_NT scope:SCOPE_SE
; GCN-NEXT: s_endpgm
@@ -89,6 +103,8 @@ entry:
define amdgpu_ps void @global_prefetch_dev_ht(ptr addrspace(1) %ptr) {
; GCN-LABEL: global_prefetch_dev_ht:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v[0:1], off th:TH_LOAD_HT scope:SCOPE_DEV
; GCN-NEXT: s_endpgm
@@ -100,6 +116,8 @@ entry:
define amdgpu_ps void @global_prefetch_sys_lu(ptr addrspace(1) %ptr) {
; GCN-LABEL: global_prefetch_sys_lu:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: global_prefetch_b8 v[0:1], off th:TH_LOAD_BYPASS scope:SCOPE_SYS
; GCN-NEXT: s_endpgm
@@ -111,6 +129,8 @@ entry:
define amdgpu_ps float @global_prefetch_and_load_b32_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GCN-LABEL: global_prefetch_and_load_b32_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll
index 1b7ea7c98f612..71788fad1c373 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll
@@ -10,6 +10,8 @@ declare void @llvm.amdgcn.global.store.async.from.lds.b128(ptr addrspace(1) %gad
define amdgpu_ps void @global_store_async_from_lds_b8_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_store_async_from_lds_b8_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_store_async_from_lds_b8 v[0:1], v2, off offset:16 th:TH_STORE_NT
@@ -17,6 +19,8 @@ define amdgpu_ps void @global_store_async_from_lds_b8_vaddr(ptr addrspace(1) %ga
;
; GFX1250-GISEL-LABEL: global_store_async_from_lds_b8_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -32,6 +36,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b8_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_store_async_from_lds_b8_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_store_async_from_lds_b8 v1, v0, s[0:1] offset:16
@@ -45,6 +51,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b32(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_store_async_from_lds_b32:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_store_async_from_lds_b32 v[0:1], v2, off offset:16 th:TH_STORE_HT scope:SCOPE_SE
@@ -52,6 +60,8 @@ define amdgpu_ps void @global_store_async_from_lds_b32(ptr addrspace(1) %gaddr,
;
; GFX1250-GISEL-LABEL: global_store_async_from_lds_b32:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -67,6 +77,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b32_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_store_async_from_lds_b32_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_store_async_from_lds_b32 v1, v0, s[0:1] offset:16
@@ -80,6 +92,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b64_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_store_async_from_lds_b64_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_store_async_from_lds_b64 v[0:1], v2, off offset:16 th:TH_STORE_NT_HT scope:SCOPE_DEV
@@ -87,6 +101,8 @@ define amdgpu_ps void @global_store_async_from_lds_b64_vaddr(ptr addrspace(1) %g
;
; GFX1250-GISEL-LABEL: global_store_async_from_lds_b64_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -102,6 +118,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b64_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_store_async_from_lds_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_store_async_from_lds_b64 v1, v0, s[0:1] offset:16
@@ -115,6 +133,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b128_vaddr(ptr addrspace(1) %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-SDAG-LABEL: global_store_async_from_lds_b128_vaddr:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 32, v[0:1]
; GFX1250-SDAG-NEXT: global_store_async_from_lds_b128 v[0:1], v2, off offset:16 th:TH_STORE_BYPASS scope:SCOPE_SYS
@@ -122,6 +142,8 @@ define amdgpu_ps void @global_store_async_from_lds_b128_vaddr(ptr addrspace(1) %
;
; GFX1250-GISEL-LABEL: global_store_async_from_lds_b128_vaddr:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 32
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -137,6 +159,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b128_saddr(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr) {
; GFX1250-LABEL: global_store_async_from_lds_b128_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 32
; GFX1250-NEXT: global_store_async_from_lds_b128 v1, v0, s[0:1] offset:16
@@ -150,6 +174,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b32_saddr_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %idx) {
; GFX1250-LABEL: global_store_async_from_lds_b32_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_store_async_from_lds_b32 v1, v0, s[0:1] offset:16 scale_offset th:TH_STORE_NT
; GFX1250-NEXT: s_endpgm
@@ -163,6 +189,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b64_saddr_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %idx) {
; GFX1250-LABEL: global_store_async_from_lds_b64_saddr_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_store_async_from_lds_b64 v1, v0, s[0:1] offset:16 scale_offset th:TH_STORE_NT
; GFX1250-NEXT: s_endpgm
@@ -176,6 +204,8 @@ entry:
define amdgpu_ps void @global_store_async_from_lds_b64_saddr_no_scale_offset(ptr addrspace(1) inreg %gaddr, ptr addrspace(3) %laddr, i32 %idx) {
; GFX1250-LABEL: global_store_async_from_lds_b64_saddr_no_scale_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
index a6264e09eeff3..5d3f11953c358 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
@@ -61,6 +61,8 @@ define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
;
; GFX1250-LABEL: is_private_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -188,6 +190,8 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
;
; GFX1250-SDAG-LABEL: is_private_sgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x4 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -266,6 +270,8 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
;
; GFX1250-GISEL-LABEL: is_private_sgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
index 1799f1c2d9556..01be410f5f9db 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
@@ -94,6 +94,8 @@ define amdgpu_kernel void @is_local_vgpr(ptr addrspace(1) %ptr.ptr) {
;
; GFX1250-LABEL: is_local_vgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -254,6 +256,8 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
;
; GFX1250-SDAG-LABEL: is_local_sgpr:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_shared_base
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x4 nv
@@ -332,6 +336,8 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
;
; GFX1250-GISEL-LABEL: is_local_sgpr:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_shared_base
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
index d544aa54adaae..fccc202a6c9ce 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
@@ -13,6 +13,8 @@ declare <4 x i32> @llvm.amdgcn.flat.load.monitor.b128.v4i32(ptr, i32, metadata)
define amdgpu_ps void @global_load_monitor_b32_vaddr_relaxed_sys(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_monitor_b32_vaddr_relaxed_sys:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_monitor_b32 v0, v[0:1], off offset:32 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -28,6 +30,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b32_saddr_relaxed_sys(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_monitor_b32_saddr_relaxed_sys:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_monitor_b32 v2, v2, s[0:1] offset:32 scope:SCOPE_SYS
@@ -44,6 +48,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b64_vaddr_acquire_agent(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_monitor_b64_vaddr_acquire_agent:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_monitor_b64 v[0:1], v[0:1], off offset:32 scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -61,6 +67,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b64_saddr_acquire_agent(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_monitor_b64_saddr_acquire_agent:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_monitor_b64 v[2:3], v2, s[0:1] offset:32 scope:SCOPE_DEV
@@ -79,6 +87,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b128_vaddr_seq_cst_workgroup(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_monitor_b128_vaddr_seq_cst_workgroup:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_monitor_b128 v[4:7], v[0:1], off offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -94,6 +104,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b128_saddr_seq_cst_workgroup(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_monitor_b128_saddr_seq_cst_workgroup:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_monitor_b128 v[2:5], v2, s[0:1] offset:32
@@ -110,6 +122,8 @@ entry:
define amdgpu_ps void @flat_load_monitor_b32_seq_cst_sys(ptr %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: flat_load_monitor_b32_seq_cst_sys:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_monitor_b32 v0, v[0:1] offset:32 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -127,6 +141,8 @@ entry:
define amdgpu_ps void @flat_load_monitor_b64_seq_cst_agent(ptr %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: flat_load_monitor_b64_seq_cst_agent:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_monitor_b64 v[0:1], v[0:1] offset:32 scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -144,6 +160,8 @@ entry:
define amdgpu_ps void @flat_load_monitor_b128_acquire_sys(ptr %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: flat_load_monitor_b128_acquire_sys:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_monitor_b128 v[4:7], v[0:1] offset:32 scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -161,6 +179,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b32_saddr_scale_offset_acquire_agent(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 %idx) {
; GFX1250-LABEL: global_load_monitor_b32_saddr_scale_offset_acquire_agent:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_monitor_b32 v2, v2, s[0:1] scale_offset scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -179,6 +199,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b64_saddr_scale_offset_acquire_workgroup(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 %idx) {
; GFX1250-LABEL: global_load_monitor_b64_saddr_scale_offset_acquire_workgroup:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_monitor_b64 v[2:3], v2, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -195,6 +217,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b64_saddr_no_scale_offset_seq_cst_sys(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use, i32 %idx) {
; GFX1250-LABEL: global_load_monitor_b64_saddr_no_scale_offset_seq_cst_sys:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -217,6 +241,8 @@ entry:
define amdgpu_ps void @global_load_monitor_b32_vaddr_monotonic_wavefront(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_monitor_b32_vaddr_monotonic_wavefront:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_monitor_b32 v0, v[0:1], off offset:32 scope:SCOPE_SE
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -233,6 +259,8 @@ entry:
define amdgpu_ps void @flat_load_monitor_b32_monotonic_singlethread(ptr %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: flat_load_monitor_b32_monotonic_singlethread:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_load_monitor_b32 v0, v[0:1] offset:32 scope:SCOPE_SE
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
index 82ac4694579ec..4870c0424e5c4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @global_load_tr4_b64_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr4_b64_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_tr4_b64 v[0:1], v[0:1], off offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -20,6 +22,8 @@ entry:
define amdgpu_ps void @global_load_tr4_b64_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr4_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_tr4_b64 v[2:3], v2, s[0:1] offset:32
@@ -36,6 +40,8 @@ entry:
define amdgpu_ps void @global_load_tr8_b64_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr8_b64_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_tr8_b64 v[0:1], v[0:1], off offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -51,6 +57,8 @@ entry:
define amdgpu_ps void @global_load_tr8_b64_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr8_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_tr8_b64 v[2:3], v2, s[0:1] offset:32
@@ -67,6 +75,8 @@ entry:
define amdgpu_ps void @global_load_tr6_b96_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr6_b96_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_tr6_b96 v[4:6], v[0:1], off offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -82,6 +92,8 @@ entry:
define amdgpu_ps void @global_load_tr6_b96_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr6_b96_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_tr6_b96 v[2:4], v2, s[0:1] offset:32
@@ -98,6 +110,8 @@ entry:
define amdgpu_ps void @global_load_tr16_b128_v8i16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr16_b128_v8i16_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_tr16_b128 v[4:7], v[0:1], off offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -113,6 +127,8 @@ entry:
define amdgpu_ps void @global_load_tr16_b128_v8i16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr16_b128_v8i16_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
@@ -129,6 +145,8 @@ entry:
define amdgpu_ps void @global_load_tr16_b128_v8f16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr16_b128_v8f16_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_tr16_b128 v[4:7], v[0:1], off offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -144,6 +162,8 @@ entry:
define amdgpu_ps void @global_load_tr16_b128_v8f16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr16_b128_v8f16_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
@@ -160,6 +180,8 @@ entry:
define amdgpu_ps void @global_load_tr16_b128_v8b16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr16_b128_v8b16_vaddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_tr16_b128 v[4:7], v[0:1], off offset:32
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -175,6 +197,8 @@ entry:
define amdgpu_ps void @global_load_tr16_b128_v8bf16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: global_load_tr16_b128_v8bf16_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
@@ -191,6 +215,8 @@ entry:
define amdgpu_ps void @ds_load_tr4_b64(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
; GFX1250-SDAG-LABEL: ds_load_tr4_b64:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: ds_load_tr4_b64 v[0:1], v0 offset:32
@@ -200,6 +226,8 @@ define amdgpu_ps void @ds_load_tr4_b64(ptr addrspace(3) %addr, ptr addrspace(1)
;
; GFX1250-GISEL-LABEL: ds_load_tr4_b64:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: ds_load_tr4_b64 v[0:1], v0 offset:32
@@ -216,6 +244,8 @@ entry:
define amdgpu_ps void @ds_load_tr4_b64_saddr(ptr addrspace(3) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: ds_load_tr4_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: ds_load_tr4_b64 v[2:3], v2 offset:32
@@ -232,6 +262,8 @@ entry:
define amdgpu_ps void @ds_load_tr8_b64(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
; GFX1250-SDAG-LABEL: ds_load_tr8_b64:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: ds_load_tr8_b64 v[0:1], v0 offset:32
@@ -241,6 +273,8 @@ define amdgpu_ps void @ds_load_tr8_b64(ptr addrspace(3) %addr, ptr addrspace(1)
;
; GFX1250-GISEL-LABEL: ds_load_tr8_b64:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: ds_load_tr8_b64 v[0:1], v0 offset:32
@@ -257,6 +291,8 @@ entry:
define amdgpu_ps void @ds_load_tr8_b64_saddr(ptr addrspace(3) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: ds_load_tr8_b64_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: ds_load_tr8_b64 v[2:3], v2 offset:32
@@ -273,6 +309,8 @@ entry:
define amdgpu_ps void @ds_load_tr6_b96(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
; GFX1250-SDAG-LABEL: ds_load_tr6_b96:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX1250-SDAG-NEXT: ds_load_tr6_b96 v[0:2], v0 offset:32
@@ -282,6 +320,8 @@ define amdgpu_ps void @ds_load_tr6_b96(ptr addrspace(3) %addr, ptr addrspace(1)
;
; GFX1250-GISEL-LABEL: ds_load_tr6_b96:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: ds_load_tr6_b96 v[0:2], v0 offset:32
@@ -298,6 +338,8 @@ entry:
define amdgpu_ps void @ds_load_tr6_b96_saddr(ptr addrspace(3) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: ds_load_tr6_b96_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: ds_load_tr6_b96 v[2:4], v2 offset:32
@@ -314,6 +356,8 @@ entry:
define amdgpu_ps void @ds_load_tr16_b128_v8i16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
; GFX1250-SDAG-LABEL: ds_load_tr16_b128_v8i16:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX1250-SDAG-NEXT: ds_load_tr16_b128 v[0:3], v0 offset:32
@@ -323,6 +367,8 @@ define amdgpu_ps void @ds_load_tr16_b128_v8i16(ptr addrspace(3) %addr, ptr addrs
;
; GFX1250-GISEL-LABEL: ds_load_tr16_b128_v8i16:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: ds_load_tr16_b128 v[0:3], v0 offset:32
@@ -339,6 +385,8 @@ entry:
define amdgpu_ps void @ds_load_tr16_b128_v8i16_saddr(ptr addrspace(3) inreg %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: ds_load_tr16_b128_v8i16_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: ds_load_tr16_b128 v[2:5], v2 offset:32
@@ -355,6 +403,8 @@ entry:
define amdgpu_ps void @ds_load_tr16_b128_v8f16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
; GFX1250-SDAG-LABEL: ds_load_tr16_b128_v8f16:
; GFX1250-SDAG: ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX1250-SDAG-NEXT: ds_load_tr16_b128 v[0:3], v0 offset:32
@@ -364,6 +414,8 @@ define amdgpu_ps void @ds_load_tr16_b128_v8f16(ptr addrspace(3) %addr, ptr addrs
;
; GFX1250-GISEL-LABEL: ds_load_tr16_b128_v8f16:
; GFX1250-GISEL: ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: ds_load_tr16_b128 v[0:3], v0 offset:32
@@ -380,6 +432,8 @@ entry:
define amdgpu_ps void @ds_load_tr16_b128_v8bf16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
; GFX1250-LABEL: ds_load_tr16_b128_v8bf16:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX1250-NEXT: ds_load_tr16_b128 v[0:3], v0 offset:32
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
index ecdcaa9b3026d..4cee48a0d419b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.amdgcn.log.bf16(bfloat) #0
define amdgpu_kernel void @log_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-LABEL: log_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -21,6 +23,8 @@ define amdgpu_kernel void @log_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; REAL16-LABEL: log_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -36,6 +40,8 @@ define amdgpu_kernel void @log_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @log_bf16_constant_4(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: log_bf16_constant_4:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_log_bf16_e32 v0, 4.0
@@ -46,6 +52,8 @@ define amdgpu_kernel void @log_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: log_bf16_constant_4:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_log_bf16_e32 v0.l, 4.0
@@ -61,6 +69,8 @@ define amdgpu_kernel void @log_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @log_bf16_constant_100(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: log_bf16_constant_100:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_log_bf16_e32 v0, 0x42c8
@@ -71,6 +81,8 @@ define amdgpu_kernel void @log_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: log_bf16_constant_100:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_log_bf16_e32 v0.l, 0x42c8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll
index e70def8469d6b..1ff355dfc5ab8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll
@@ -68,6 +68,8 @@ define void @test_perm_pk16_b8_u4(i64 %a, i64 %b, <2 x i32> %c, ptr %out) {
define amdgpu_ps <2 x i32> @test_perm_pk16_b4_u4_sss(i32 inreg %a, i32 inreg %b, <2 x i32> inreg %c) {
; GFX1250-SDAG-LABEL: test_perm_pk16_b4_u4_sss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -79,6 +81,8 @@ define amdgpu_ps <2 x i32> @test_perm_pk16_b4_u4_sss(i32 inreg %a, i32 inreg %b,
;
; GFX1250-GISEL-LABEL: test_perm_pk16_b4_u4_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -94,6 +98,8 @@ define amdgpu_ps <2 x i32> @test_perm_pk16_b4_u4_sss(i32 inreg %a, i32 inreg %b,
define amdgpu_ps <3 x i32> @test_perm_pk16_b6_u4_sss(i32 inreg %a, i64 inreg %b, <2 x i32> inreg %c) {
; GFX1250-SDAG-LABEL: test_perm_pk16_b6_u4_sss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s4
; GFX1250-SDAG-NEXT: s_mov_b32 s7, s2
@@ -109,6 +115,8 @@ define amdgpu_ps <3 x i32> @test_perm_pk16_b6_u4_sss(i32 inreg %a, i64 inreg %b,
;
; GFX1250-GISEL-LABEL: test_perm_pk16_b6_u4_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_mov_b32 s6, s3
; GFX1250-GISEL-NEXT: s_mov_b32 s7, s4
@@ -130,6 +138,8 @@ define amdgpu_ps <3 x i32> @test_perm_pk16_b6_u4_sss(i32 inreg %a, i64 inreg %b,
define amdgpu_ps <4 x i32> @test_perm_pk16_b8_u4_sss(i64 inreg %a, i64 inreg %b, <2 x i32> inreg %c) {
; GFX1250-SDAG-LABEL: test_perm_pk16_b8_u4_sss:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -144,6 +154,8 @@ define amdgpu_ps <4 x i32> @test_perm_pk16_b8_u4_sss(i64 inreg %a, i64 inreg %b,
;
; GFX1250-GISEL-LABEL: test_perm_pk16_b8_u4_sss:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
index f1702901111d3..6dc91c74abace 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
@@ -31,6 +31,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i32(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_atomic_buffer_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -118,6 +120,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i32_off(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_atomic_buffer_load_i32_off:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -204,6 +208,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i32_soff(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_atomic_buffer_load_i32_soff:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -293,6 +299,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i32_dlc(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_atomic_buffer_load_i32_dlc:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -381,6 +389,8 @@ define amdgpu_kernel void @raw_nonatomic_buffer_load_i32(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_nonatomic_buffer_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -471,6 +481,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
;
; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_i64:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -492,6 +504,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
;
; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_i64:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -513,6 +527,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
;
; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_i64:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -534,6 +550,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
;
; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_i64:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -585,6 +603,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v2i16(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_atomic_buffer_load_v2i16:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -758,6 +778,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
;
; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -779,6 +801,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
;
; GFX12-SDAG-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -802,6 +826,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
;
; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -825,6 +851,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
;
; GFX12-GISEL-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -879,6 +907,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_v4i32(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_atomic_buffer_load_v4i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -969,6 +999,8 @@ define amdgpu_kernel void @raw_atomic_buffer_load_ptr(<4 x i32> %addr) {
;
; GFX12-LABEL: raw_atomic_buffer_load_ptr:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll
index 584971fba535d..76858af2dcc0d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll
@@ -67,6 +67,8 @@ define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspa
;
; GFX1250-LABEL: raw_buffer_load_i8_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -143,6 +145,8 @@ define amdgpu_ps void @raw_buffer_load_i16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
;
; GFX1250-LABEL: raw_buffer_load_i16_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -219,6 +223,8 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
;
; GFX1250-LABEL: raw_buffer_load_f16_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -295,6 +301,8 @@ define amdgpu_ps void @raw_buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrsp
;
; GFX1250-LABEL: raw_buffer_load_i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -391,6 +399,8 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
;
; GFX1250-LABEL: raw_buffer_load_v2i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -487,6 +497,8 @@ define amdgpu_ps void @raw_buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addr
;
; GFX1250-LABEL: raw_buffer_load_v2f32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -589,6 +601,8 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
;
; GFX1250-LABEL: raw_buffer_load_v3i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -692,6 +706,8 @@ define amdgpu_ps void @raw_buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addr
;
; GFX1250-LABEL: raw_buffer_load_v3f32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -782,6 +798,8 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
;
; GFX1250-LABEL: raw_buffer_load_v4i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -872,6 +890,8 @@ define amdgpu_ps void @raw_buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addr
;
; GFX1250-LABEL: raw_buffer_load_v4f32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
index 5559d1e383aab..78b3a6b31a0f0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
@@ -31,6 +31,8 @@ define amdgpu_ps void @buffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <
;
; GFX1250-LABEL: buffer_store:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
@@ -62,6 +64,8 @@ define amdgpu_ps void @buffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
;
; GFX1250-LABEL: buffer_store_immoffs:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:42
; GFX1250-NEXT: s_endpgm
@@ -88,6 +92,8 @@ define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float>, i32) {
;
; GFX1250-LABEL: buffer_store_ofs:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -133,6 +139,8 @@ define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float>, i32, i32,
;
; GFX1250-LABEL: buffer_store_wait:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null offen
@@ -165,6 +173,8 @@ define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %
;
; GFX1250-LABEL: buffer_store_x1:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -191,6 +201,8 @@ define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data,
;
; GFX1250-LABEL: buffer_store_x2:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -222,6 +234,8 @@ define amdgpu_ps void @buffer_store_x1_offen_merged_and(<4 x i32> inreg %rsrc, i
;
; GFX1250-LABEL: buffer_store_x1_offen_merged_and:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_add_nc_u32 v7, 4, v0 :: v_dual_add_nc_u32 v8, 8, v0
; GFX1250-NEXT: v_dual_add_nc_u32 v9, 12, v0 :: v_dual_add_nc_u32 v10, 16, v0
@@ -275,6 +289,8 @@ define amdgpu_ps void @buffer_store_x1_offen_merged_or(<4 x i32> inreg %rsrc, i3
;
; GFX1250-LABEL: buffer_store_x1_offen_merged_or:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v7, v6 :: v_dual_mov_b32 v6, v5
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
@@ -326,6 +342,8 @@ define amdgpu_ps void @buffer_store_x1_offen_merged_glc_slc(<4 x i32> inreg %rsr
;
; GFX1250-LABEL: buffer_store_x1_offen_merged_glc_slc:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_add_nc_u32 v7, 4, v0 :: v_dual_add_nc_u32 v8, 8, v0
; GFX1250-NEXT: v_dual_add_nc_u32 v9, 12, v0 :: v_dual_add_nc_u32 v10, 16, v0
@@ -371,6 +389,8 @@ define amdgpu_ps void @buffer_store_x2_offen_merged_and(<4 x i32> inreg %rsrc, i
;
; GFX1250-LABEL: buffer_store_x2_offen_merged_and:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -407,6 +427,8 @@ define amdgpu_ps void @buffer_store_x2_offen_merged_or(<4 x i32> inreg %rsrc, i3
;
; GFX1250-LABEL: buffer_store_x2_offen_merged_or:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
@@ -444,6 +466,8 @@ define amdgpu_ps void @buffer_store_x1_offset_merged(<4 x i32> inreg %rsrc, floa
;
; GFX1250-LABEL: buffer_store_x1_offset_merged:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:4
@@ -476,6 +500,8 @@ define amdgpu_ps void @buffer_store_x2_offset_merged(<4 x i32> inreg %rsrc, <2 x
;
; GFX1250-LABEL: buffer_store_x2_offset_merged:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:4
; GFX1250-NEXT: s_endpgm
@@ -510,6 +536,8 @@ define amdgpu_ps void @buffer_store_int(<4 x i32> inreg, <4 x i32>, <2 x i32>, i
;
; GFX1250-LABEL: buffer_store_int:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
@@ -544,6 +572,8 @@ define amdgpu_ps void @raw_buffer_store_byte(<4 x i32> inreg %rsrc, float %v1) {
;
; GFX1250-LABEL: raw_buffer_store_byte:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX1250-NEXT: buffer_store_b8 v0, off, s[0:3], null
@@ -576,6 +606,8 @@ define amdgpu_ps void @raw_buffer_store_short(<4 x i32> inreg %rsrc, float %v1)
;
; GFX1250-LABEL: raw_buffer_store_short:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX1250-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -605,6 +637,8 @@ define amdgpu_ps void @raw_buffer_store_f16(<4 x i32> inreg %rsrc, i32 %v1) {
;
; GFX1250-LABEL: raw_buffer_store_f16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-NEXT: s_endpgm
@@ -633,6 +667,8 @@ define amdgpu_ps void @buffer_store_v2f16(<4 x i32> inreg %rsrc, <2 x half> %dat
;
; GFX1250-LABEL: buffer_store_v2f16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -659,6 +695,8 @@ define amdgpu_ps void @buffer_store_v4f16(<4 x i32> inreg %rsrc, <4 x half> %dat
;
; GFX1250-LABEL: buffer_store_v4f16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -685,6 +723,8 @@ define amdgpu_ps void @raw_buffer_store_i16(<4 x i32> inreg %rsrc, i32 %v1) {
;
; GFX1250-LABEL: raw_buffer_store_i16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-NEXT: s_endpgm
@@ -712,6 +752,8 @@ define amdgpu_ps void @buffer_store_v2i16(<4 x i32> inreg %rsrc, <2 x i16> %data
;
; GFX1250-LABEL: buffer_store_v2i16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -738,6 +780,8 @@ define amdgpu_ps void @buffer_store_v4i16(<4 x i32> inreg %rsrc, <4 x i16> %data
;
; GFX1250-LABEL: buffer_store_v4i16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -769,6 +813,8 @@ define amdgpu_ps void @raw_buffer_store_x1_offset_merged(<4 x i32> inreg %rsrc,
;
; GFX1250-LABEL: raw_buffer_store_x1_offset_merged:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:4
@@ -814,6 +860,8 @@ define amdgpu_ps void @raw_buffer_store_x1_offset_swizzled_not_merged_pregfx12(<
;
; GFX1250-LABEL: raw_buffer_store_x1_offset_swizzled_not_merged_pregfx12:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:4 scope:SCOPE_SE
@@ -855,6 +903,8 @@ define amdgpu_ps void @raw_buffer_store_x1_offset_swizzled_not_merged(<4 x i32>
;
; GFX1250-LABEL: raw_buffer_store_x1_offset_swizzled_not_merged:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x5
; GFX1250-NEXT: buffer_store_b32 v0, off, s[0:3], null offset:4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
index f690506b974f7..eb06520422c77 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
@@ -31,6 +31,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_ptr_load_i32(ptr addrspace(8) %
;
; GFX12-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -118,6 +120,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %
;
; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_off:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -204,6 +208,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8)
;
; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -293,6 +299,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %
;
; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -381,6 +389,8 @@ define amdgpu_kernel void @raw_nonptr_atomic_buffer_load_i32(ptr addrspace(8) %p
;
; GFX12-LABEL: raw_nonptr_atomic_buffer_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -471,6 +481,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr)
;
; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -492,6 +504,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr)
;
; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -513,6 +527,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr)
;
; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_i64:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -534,6 +550,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr)
;
; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_i64:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -585,6 +603,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %pt
;
; GFX12-LABEL: raw_ptr_atomic_buffer_load_v2i16:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -758,6 +778,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
;
; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -779,6 +801,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
;
; GFX12-SDAG-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -802,6 +826,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
;
; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -825,6 +851,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %pt
;
; GFX12-GISEL-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -879,6 +907,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %pt
;
; GFX12-LABEL: raw_ptr_atomic_buffer_load_v4i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -969,6 +999,8 @@ define amdgpu_kernel void @raw_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr)
;
; GFX12-LABEL: raw_ptr_atomic_buffer_load_ptr:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
index 12997dfa5fabe..f58373e871450 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -36,6 +36,8 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load(ptr addrsp
;
; GFX1250-LABEL: buffer_load:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
@@ -82,6 +84,8 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load_dlc(ptr ad
;
; GFX1250-LABEL: buffer_load_dlc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null th:TH_LOAD_NT_RT
@@ -132,6 +136,8 @@ define amdgpu_ps {<4 x float>, <4 x float>, <4 x float>} @buffer_load_volatile(p
;
; GFX1250-LABEL: buffer_load_volatile:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -171,6 +177,8 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs(ptr addrspace(8) inreg) {
;
; GFX1250-LABEL: buffer_load_immoffs:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:40
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -204,6 +212,8 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg)
;
; GFX1250-LABEL: buffer_load_immoffs_large:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_movk_i32 s4, 0x1ffc
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], s4 offset:4
@@ -235,6 +245,8 @@ define amdgpu_ps <4 x float> @buffer_load_ofs(ptr addrspace(8) inreg, i32) {
;
; GFX1250-LABEL: buffer_load_ofs:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -265,6 +277,8 @@ define amdgpu_ps <4 x float> @buffer_load_ofs_imm(ptr addrspace(8) inreg, i32) {
;
; GFX1250-LABEL: buffer_load_ofs_imm:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u32_e32 v0, 60, v0
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
@@ -297,6 +311,8 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_12bit(ptr addrspace(8) i
;
; GFX1250-LABEL: buffer_load_voffset_large_12bit:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:4092
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -330,6 +346,8 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_13bit(ptr addrspace(8) i
;
; GFX1250-LABEL: buffer_load_voffset_large_13bit:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:8188
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -363,6 +381,8 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_16bit(ptr addrspace(8) i
;
; GFX1250-LABEL: buffer_load_voffset_large_16bit:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:65532
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -396,6 +416,8 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_23bit(ptr addrspace(8) i
;
; GFX1250-LABEL: buffer_load_voffset_large_23bit:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null offset:8388604
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -429,6 +451,8 @@ define amdgpu_ps <4 x float> @buffer_load_voffset_large_24bit(ptr addrspace(8) i
;
; GFX1250-LABEL: buffer_load_voffset_large_24bit:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen offset:8388604
@@ -461,6 +485,8 @@ define amdgpu_ps float @buffer_load_x1(ptr addrspace(8) inreg %rsrc, i32 %ofs) {
;
; GFX1250-LABEL: buffer_load_x1:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -491,6 +517,8 @@ define amdgpu_ps <2 x float> @buffer_load_x2(ptr addrspace(8) inreg %rsrc, i32 %
;
; GFX1250-LABEL: buffer_load_x2:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -517,6 +545,8 @@ define amdgpu_ps <4 x float> @buffer_load_negative_offset(ptr addrspace(8) inreg
;
; GFX1250-LABEL: buffer_load_negative_offset:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u32_e32 v0, -16, v0
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen
@@ -551,6 +581,8 @@ define amdgpu_ps float @buffer_load_mmo(ptr addrspace(8) inreg %rsrc, ptr addrsp
;
; GFX1250-LABEL: buffer_load_mmo:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b32 v1, off, s[0:3], null
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -596,6 +628,8 @@ define amdgpu_ps {<4 x float>, <2 x float>, float} @buffer_load_int(ptr addrspac
;
; GFX1250-LABEL: buffer_load_int:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: buffer_load_b128 v[0:3], off, s[0:3], null
@@ -640,6 +674,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_ubyte(ptr addrspace(8) inreg %rsrc)
;
; GFX1250-LABEL: raw_ptr_buffer_load_ubyte:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_u8 v0, off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -676,6 +712,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_i16(ptr addrspace(8) inreg %rsrc) {
;
; GFX1250-LABEL: raw_ptr_buffer_load_i16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_u16 v0, off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -712,6 +750,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_sbyte(ptr addrspace(8) inreg %rsrc)
;
; GFX1250-LABEL: raw_ptr_buffer_load_sbyte:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_i8 v0, off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -748,6 +788,8 @@ define amdgpu_ps float @raw_ptr_buffer_load_sshort(ptr addrspace(8) inreg %rsrc)
;
; GFX1250-LABEL: raw_ptr_buffer_load_sshort:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_i16 v0, off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -785,6 +827,8 @@ define amdgpu_ps void @raw_ptr_buffer_load_f16(ptr addrspace(8) inreg %rsrc, ptr
;
; GFX1250-LABEL: raw_ptr_buffer_load_f16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_u16 v1, off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -821,6 +865,8 @@ define amdgpu_ps void @raw_ptr_buffer_load_v2f16(ptr addrspace(8) inreg %rsrc, p
;
; GFX1250-LABEL: raw_ptr_buffer_load_v2f16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b32 v1, off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -857,6 +903,8 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4f16(ptr addrspace(8) inreg %rsrc, p
;
; GFX1250-LABEL: raw_ptr_buffer_load_v4f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -891,6 +939,8 @@ define amdgpu_ps void @raw_ptr_buffer_load_v8f16(ptr addrspace(8) inreg %rsrc, p
;
; GFX1250-LABEL: raw_ptr_buffer_load_v8f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -926,6 +976,8 @@ define amdgpu_ps void @raw_ptr_buffer_load_v2i16(ptr addrspace(8) inreg %rsrc, p
;
; GFX1250-LABEL: raw_ptr_buffer_load_v2i16:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b32 v1, off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -962,6 +1014,8 @@ define amdgpu_ps void @raw_ptr_buffer_load_v4i16(ptr addrspace(8) inreg %rsrc, p
;
; GFX1250-LABEL: raw_ptr_buffer_load_v4i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b64 v[2:3], off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -996,6 +1050,8 @@ define amdgpu_ps void @raw_ptr_buffer_load_v8i16(ptr addrspace(8) inreg %rsrc, p
;
; GFX1250-LABEL: raw_ptr_buffer_load_v8i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_load_b128 v[2:5], off, s[0:3], null
; GFX1250-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll
index 83d1e070b10e6..1465c1335e774 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll
@@ -34,6 +34,8 @@ define amdgpu_ps void @buffer_store_bf16(ptr addrspace(8) inreg %rsrc, bfloat %d
;
; GFX12-LABEL: buffer_store_bf16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b16 v0, v1, s[0:3], null offen
; GFX12-NEXT: s_endpgm
@@ -69,6 +71,8 @@ define amdgpu_ps void @buffer_store_v2bf16(ptr addrspace(8) inreg %rsrc, <2 x bf
;
; GFX12-LABEL: buffer_store_v2bf16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b32 v0, v1, s[0:3], null offen
; GFX12-NEXT: s_endpgm
@@ -104,6 +108,8 @@ define amdgpu_ps void @buffer_store_v4bf16(ptr addrspace(8) inreg %rsrc, <4 x bf
;
; GFX12-LABEL: buffer_store_v4bf16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null offen
; GFX12-NEXT: s_endpgm
@@ -145,6 +151,8 @@ define amdgpu_ps void @buffer_store_v8bf16(ptr addrspace(8) inreg %rsrc, <8 x bf
;
; GFX12-LABEL: buffer_store_v8bf16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null offen
; GFX12-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
index d3cce593be49f..417e6094ad275 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.amdgcn.rcp.bf16(bfloat) #0
define amdgpu_kernel void @rcp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; SDAG-TRUE16-LABEL: rcp_bf16:
; SDAG-TRUE16: ; %bb.0:
+; SDAG-TRUE16-NEXT: global_wb
+; SDAG-TRUE16-NEXT: v_nop
; SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -21,6 +23,8 @@ define amdgpu_kernel void @rcp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; SDAG-FAKE16-LABEL: rcp_bf16:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -36,6 +40,8 @@ define amdgpu_kernel void @rcp_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @rcp_bf16_constant_4(ptr addrspace(1) %out) #1 {
; SDAG-TRUE16-LABEL: rcp_bf16_constant_4:
; SDAG-TRUE16: ; %bb.0:
+; SDAG-TRUE16-NEXT: global_wb
+; SDAG-TRUE16-NEXT: v_nop
; SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -46,6 +52,8 @@ define amdgpu_kernel void @rcp_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; SDAG-FAKE16-LABEL: rcp_bf16_constant_4:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3e80
@@ -60,6 +68,8 @@ define amdgpu_kernel void @rcp_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @rcp_bf16_constant_100(ptr addrspace(1) %out) #1 {
; SDAG-TRUE16-LABEL: rcp_bf16_constant_100:
; SDAG-TRUE16: ; %bb.0:
+; SDAG-TRUE16-NEXT: global_wb
+; SDAG-TRUE16-NEXT: v_nop
; SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -70,6 +80,8 @@ define amdgpu_kernel void @rcp_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; SDAG-FAKE16-LABEL: rcp_bf16_constant_100:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x3c24
@@ -84,6 +96,8 @@ define amdgpu_kernel void @rcp_bf16_constant_100(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @rcp_undef_bf16(ptr addrspace(1) %out) #1 {
; SDAG-TRUE16-LABEL: rcp_undef_bf16:
; SDAG-TRUE16: ; %bb.0:
+; SDAG-TRUE16-NEXT: global_wb
+; SDAG-TRUE16-NEXT: v_nop
; SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -94,6 +108,8 @@ define amdgpu_kernel void @rcp_undef_bf16(ptr addrspace(1) %out) #1 {
;
; SDAG-FAKE16-LABEL: rcp_undef_bf16:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0x7fc0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
index 8f9a77d4ea2fd..02b11f2ca869a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.amdgcn.rsq.bf16(bfloat) #0
define amdgpu_kernel void @rsq_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; SDAG-REAL16-LABEL: rsq_bf16:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -21,6 +23,8 @@ define amdgpu_kernel void @rsq_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; SDAG-FAKE16-LABEL: rsq_bf16:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -36,6 +40,8 @@ define amdgpu_kernel void @rsq_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @rsq_bf16_constant_4(ptr addrspace(1) %out) #1 {
; SDAG-REAL16-LABEL: rsq_bf16_constant_4:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 4.0
@@ -46,6 +52,8 @@ define amdgpu_kernel void @rsq_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; SDAG-FAKE16-LABEL: rsq_bf16_constant_4:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_rsq_bf16_e32 v0, 4.0
@@ -61,6 +69,8 @@ define amdgpu_kernel void @rsq_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @rsq_bf16_constant_100(ptr addrspace(1) %out) #1 {
; SDAG-REAL16-LABEL: rsq_bf16_constant_100:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-REAL16-NEXT: v_rsq_bf16_e32 v0.l, 0x42c8
@@ -71,6 +81,8 @@ define amdgpu_kernel void @rsq_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; SDAG-FAKE16-LABEL: rsq_bf16_constant_100:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; SDAG-FAKE16-NEXT: v_rsq_bf16_e32 v0, 0x42c8
@@ -86,11 +98,15 @@ define amdgpu_kernel void @rsq_bf16_constant_100(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @rsq_undef_bf16(ptr addrspace(1) %out) #1 {
; SDAG-REAL16-LABEL: rsq_undef_bf16:
; SDAG-REAL16: ; %bb.0:
+; SDAG-REAL16-NEXT: global_wb
+; SDAG-REAL16-NEXT: v_nop
; SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-REAL16-NEXT: s_endpgm
;
; SDAG-FAKE16-LABEL: rsq_undef_bf16:
; SDAG-FAKE16: ; %bb.0:
+; SDAG-FAKE16-NEXT: global_wb
+; SDAG-FAKE16-NEXT: v_nop
; SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-FAKE16-NEXT: s_endpgm
%rsq = call bfloat @llvm.amdgcn.rsq.bf16(bfloat undef)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
index aa288b82476ea..afd255c0be969 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
@@ -35,6 +35,8 @@ define amdgpu_ps i32 @s_buffer_load_imm(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x4 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -70,6 +72,8 @@ define amdgpu_ps i32 @s_buffer_load_index(<4 x i32> inreg %desc, i32 inreg %inde
;
; GFX1250-LABEL: s_buffer_load_index:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -102,6 +106,8 @@ define amdgpu_ps i32 @s_buffer_load_index_divergent(<4 x i32> inreg %desc, i32 %
;
; GFX1250-LABEL: s_buffer_load_index_divergent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
@@ -135,6 +141,8 @@ define amdgpu_ps i32 @s_buffer_load_index_divergent_offset(<4 x i32> inreg %desc
;
; GFX1250-LABEL: s_buffer_load_index_divergent_offset:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: v_add_nc_u32_e32 v0, 32, v0
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -170,6 +178,8 @@ define amdgpu_ps i32 @s_buffer_load_index_divergent_offset_nuw(<4 x i32> inreg %
;
; GFX1250-LABEL: s_buffer_load_index_divergent_offset_nuw:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: buffer_load_b32 v0, v0, s[0:3], null offen offset:32 nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
@@ -207,6 +217,8 @@ define amdgpu_ps <2 x i32> @s_buffer_loadx2_imm(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_loadx2_imm:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], 0x40 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -257,6 +269,8 @@ define amdgpu_ps float @s_buffer_loadx2_index(<4 x i32> inreg %desc, i32 inreg %
;
; GFX1250-LABEL: s_buffer_loadx2_index:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b64 s[0:1], s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_add_f32 s0, s0, s1
@@ -299,6 +313,8 @@ define amdgpu_ps <2 x i32> @s_buffer_loadx2_index_divergent(<4 x i32> inreg %des
;
; GFX1250-LABEL: s_buffer_loadx2_index_divergent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
@@ -336,6 +352,8 @@ define amdgpu_ps <3 x i32> @s_buffer_loadx3_imm(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_loadx3_imm:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b96 s[0:2], s[0:3], 0x40 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -371,6 +389,8 @@ define amdgpu_ps <3 x i32> @s_buffer_loadx3_index(<4 x i32> inreg %desc, i32 inr
;
; GFX1250-LABEL: s_buffer_loadx3_index:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b96 s[0:2], s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -418,6 +438,8 @@ define amdgpu_ps <3 x i32> @s_buffer_loadx3_index_divergent(<4 x i32> inreg %des
;
; GFX1250-LABEL: s_buffer_loadx3_index_divergent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: buffer_load_b96 v[0:2], v0, s[0:3], null offen nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
@@ -456,6 +478,8 @@ define amdgpu_ps <4 x i32> @s_buffer_loadx4_imm(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_loadx4_imm:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], 0xc8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -491,6 +515,8 @@ define amdgpu_ps <4 x i32> @s_buffer_loadx4_index(<4 x i32> inreg %desc, i32 inr
;
; GFX1250-LABEL: s_buffer_loadx4_index:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b128 s[0:3], s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -532,6 +558,8 @@ define amdgpu_ps <4 x i32> @s_buffer_loadx4_index_divergent(<4 x i32> inreg %des
;
; GFX1250-LABEL: s_buffer_loadx4_index_divergent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: buffer_load_b128 v[0:3], v0, s[0:3], null offen nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
@@ -579,6 +607,8 @@ define amdgpu_ps <2 x i32> @s_buffer_load_imm_mergex2(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_mergex2:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b64 s[4:5], s[0:3], 0x4 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, s4
@@ -629,6 +659,8 @@ define amdgpu_ps <4 x i32> @s_buffer_load_imm_mergex4(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_mergex4:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b128 s[4:7], s[0:3], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b32 s0, s4
@@ -770,6 +802,8 @@ define amdgpu_ps i32 @s_buffer_load_index_across_bb(<4 x i32> inreg %desc, i32 %
;
; GFX1250-LABEL: s_buffer_load_index_across_bb:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_get_pc_i64 s[4:5]
; GFX1250-NEXT: s_add_nc_u64 s[4:5], s[4:5], gv at gotpcrel+4
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 4, v0
@@ -823,6 +857,8 @@ define amdgpu_ps <2 x i32> @s_buffer_load_index_across_bb_merged(<4 x i32> inreg
;
; GFX1250-LABEL: s_buffer_load_index_across_bb_merged:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX1250-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen offset:8 nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -882,6 +918,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_neg1(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_neg1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s4, -1
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -935,6 +973,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_neg4(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_neg4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s4, -4
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -988,6 +1028,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_neg8(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_neg8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s4, -8
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1041,6 +1083,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_bit31(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_bit31:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_brev_b32 s4, 1
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1094,6 +1138,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_bit30(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_bit30:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s4, 2.0
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1147,6 +1193,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_bit29(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_bit29:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_brev_b32 s4, 4
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1199,6 +1247,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_bit21(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_bit21:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x200000 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1250,6 +1300,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_bit20(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_bit20:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x100000 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1302,6 +1354,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_neg_bit20(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_neg_bit20:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s4, 0xfff00000
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1345,6 +1399,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_bit19(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_bit19:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x80000 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1397,6 +1453,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_neg_bit19(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_neg_bit19:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_mov_b32 s4, 0xfff80000
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], s4 offset:0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1441,6 +1499,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_255(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_255:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0xff nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1475,6 +1535,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_256(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_256:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x100 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1509,6 +1571,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_1016(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_1016:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x3f8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1543,6 +1607,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_1020(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_1020:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x3fc nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1586,6 +1652,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_1021(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_1021:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x3fd nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1628,6 +1696,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_1024(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_1024:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x400 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1671,6 +1741,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_1025(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_1025:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x401 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -1713,6 +1785,8 @@ define amdgpu_ps i32 @s_buffer_load_imm_1028(<4 x i32> inreg %desc) {
;
; GFX1250-LABEL: s_buffer_load_imm_1028:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_buffer_load_b32 s0, s[0:3], 0x400 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.prefetch.inst.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.prefetch.inst.ll
index e546e52f4fec7..b93f5eb10e7a1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.prefetch.inst.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.prefetch.inst.ll
@@ -12,6 +12,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_sgpr_len(ptr addrspace(4) inreg %
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_sgpr_len:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, s2, 0
; GFX1250-NEXT: s_endpgm
@@ -27,6 +29,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_imm_base_sgpr_len(ptr addrspace(4) inr
;
; GFX1250-LABEL: prefetch_inst_sgpr_imm_base_sgpr_len:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x200, s2, 0
; GFX1250-NEXT: s_endpgm
@@ -43,6 +47,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_imm_len(ptr addrspace(4) inreg %p
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_imm_len:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, null, 31
; GFX1250-NEXT: s_endpgm
@@ -58,6 +64,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_imm_base_imm_len(ptr addrspace(4) inre
;
; GFX1250-LABEL: prefetch_inst_sgpr_imm_base_imm_len:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x200, null, 31
; GFX1250-NEXT: s_endpgm
@@ -76,6 +84,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_sgpr_len(ptr addrspace(4) %ptr, i
;
; GFX1250-LABEL: prefetch_inst_vgpr_base_sgpr_len:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v1
@@ -106,6 +116,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_imm_base_sgpr_len(ptr addrspace(4) %pt
;
; GFX1250-SDAG-LABEL: prefetch_inst_vgpr_imm_base_sgpr_len:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v1
@@ -114,6 +126,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_imm_base_sgpr_len(ptr addrspace(4) %pt
;
; GFX1250-GISEL-LABEL: prefetch_inst_vgpr_imm_base_sgpr_len:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x200, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -137,6 +151,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_vgpr_len(ptr addrspace(4) inreg %
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_vgpr_len:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, s2, 0
@@ -155,6 +171,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_imm_len(ptr addrspace(4) %ptr) {
;
; GFX1250-LABEL: prefetch_inst_vgpr_base_imm_len:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
@@ -183,6 +201,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_vgpr_len(ptr addrspace(4) %ptr, i
;
; GFX1250-SDAG-LABEL: prefetch_inst_vgpr_base_vgpr_len:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v2
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
@@ -192,6 +212,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_vgpr_len(ptr addrspace(4) %ptr, i
;
; GFX1250-GISEL-LABEL: prefetch_inst_vgpr_base_vgpr_len:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v1
@@ -210,6 +232,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_sgpr_len_global(ptr addrspace(1)
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_sgpr_len_global:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, s2, 0
; GFX1250-NEXT: s_endpgm
@@ -225,6 +249,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_imm_base_sgpr_len_global(ptr addrspace
;
; GFX1250-LABEL: prefetch_inst_sgpr_imm_base_sgpr_len_global:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x200, s2, 0
; GFX1250-NEXT: s_endpgm
@@ -241,6 +267,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_imm_len_global(ptr addrspace(1) i
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_imm_len_global:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, null, 31
; GFX1250-NEXT: s_endpgm
@@ -256,6 +284,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_imm_base_imm_len_global(ptr addrspace(
;
; GFX1250-LABEL: prefetch_inst_sgpr_imm_base_imm_len_global:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x200, null, 31
; GFX1250-NEXT: s_endpgm
@@ -274,6 +304,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_sgpr_len_global(ptr addrspace(1)
;
; GFX1250-LABEL: prefetch_inst_vgpr_base_sgpr_len_global:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v1
@@ -293,6 +325,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_imm_len_global(ptr addrspace(1) %
;
; GFX1250-LABEL: prefetch_inst_vgpr_base_imm_len_global:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
@@ -311,6 +345,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_vgpr_len_global(ptr addrspace(1)
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_vgpr_len_global:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, s2, 0
@@ -338,6 +374,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_vgpr_len_global(ptr addrspace(1)
;
; GFX1250-SDAG-LABEL: prefetch_inst_vgpr_base_vgpr_len_global:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v2
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
@@ -347,6 +385,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_vgpr_len_global(ptr addrspace(1)
;
; GFX1250-GISEL-LABEL: prefetch_inst_vgpr_base_vgpr_len_global:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v1
@@ -365,6 +405,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_sgpr_len_flat(ptr inreg %ptr, i32
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_sgpr_len_flat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, s2, 0
; GFX1250-NEXT: s_endpgm
@@ -380,6 +422,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_imm_base_sgpr_len_flat(ptr inreg %ptr,
;
; GFX1250-LABEL: prefetch_inst_sgpr_imm_base_sgpr_len_flat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x200, s2, 0
; GFX1250-NEXT: s_endpgm
@@ -396,6 +440,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_imm_len_flat(ptr inreg %ptr) {
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_imm_len_flat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, null, 31
; GFX1250-NEXT: s_endpgm
@@ -411,6 +457,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_imm_base_imm_len_flat(ptr inreg %ptr)
;
; GFX1250-LABEL: prefetch_inst_sgpr_imm_base_imm_len_flat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x200, null, 31
; GFX1250-NEXT: s_endpgm
@@ -429,6 +477,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_sgpr_len_flat(ptr %ptr, i32 inreg
;
; GFX1250-LABEL: prefetch_inst_vgpr_base_sgpr_len_flat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: v_readfirstlane_b32 s3, v1
@@ -448,6 +498,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_imm_len_flat(ptr %ptr) {
;
; GFX1250-LABEL: prefetch_inst_vgpr_base_imm_len_flat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
@@ -466,6 +518,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_base_vgpr_len_flat(ptr inreg %ptr, i32
;
; GFX1250-LABEL: prefetch_inst_sgpr_base_vgpr_len_flat:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-NEXT: s_prefetch_inst s[0:1], 0x0, s2, 0
@@ -493,6 +547,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_vgpr_len_flat(ptr %ptr, i32 %len)
;
; GFX1250-SDAG-LABEL: prefetch_inst_vgpr_base_vgpr_len_flat:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v2
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v0
@@ -502,6 +558,8 @@ define amdgpu_ps void @prefetch_inst_vgpr_base_vgpr_len_flat(ptr %ptr, i32 %len)
;
; GFX1250-GISEL-LABEL: prefetch_inst_vgpr_base_vgpr_len_flat:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v1
@@ -526,6 +584,8 @@ define amdgpu_ps void @prefetch_inst_function_target() {
;
; GFX1250-LABEL: prefetch_inst_function_target:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_get_pc_i64 s[0:1]
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], target at gotpcrel+4
@@ -549,6 +609,8 @@ define amdgpu_ps void @prefetch_inst_blockaddress() {
;
; GFX1250-LABEL: prefetch_inst_blockaddress:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_get_pc_i64 s[0:1]
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], .Ltmp0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx1250.ll
index 6c6a04b1dfa73..292138bcf8fb0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx1250.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @test_asynccnt() {
; GFX12-LABEL: test_asynccnt:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_wait_asynccnt 0x0
; GFX12-NEXT: s_endpgm
@@ -15,6 +17,8 @@ define amdgpu_ps void @test_asynccnt() {
define amdgpu_ps void @test_tensorcnt() {
; GFX12-LABEL: test_tensorcnt:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_wait_tensorcnt 0x0
; GFX12-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
index 9616f17f25c81..eb09b3a719fd2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll
@@ -104,6 +104,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_SWMMAC_cluster(ptr
;
; COEXEC-LABEL: test_sched_group_barrier_pipeline_SWMMAC_cluster:
; COEXEC: ; %bb.0: ; %entry
+; COEXEC-NEXT: global_wb
+; COEXEC-NEXT: v_nop
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; COEXEC-NEXT: v_dual_mov_b32 v48, 0 :: v_dual_lshlrev_b32 v0, 4, v0
@@ -321,6 +323,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_SWMMAC_interleaved(
;
; COEXEC-LABEL: test_sched_group_barrier_pipeline_SWMMAC_interleaved:
; COEXEC: ; %bb.0: ; %entry
+; COEXEC-NEXT: global_wb
+; COEXEC-NEXT: v_nop
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; COEXEC-NEXT: v_mov_b32_e32 v16, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sendmsg.rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sendmsg.rtn.ll
index b8fb7cc43179b..5e0695e7bfc39 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sendmsg.rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sendmsg.rtn.ll
@@ -25,6 +25,8 @@ define amdgpu_kernel void @test_get_doorbell(ptr addrspace(1) %out) {
;
; GFX1250-SDAG-LABEL: test_get_doorbell:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DOORBELL)
@@ -35,6 +37,8 @@ define amdgpu_kernel void @test_get_doorbell(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-LABEL: test_get_doorbell:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DOORBELL)
@@ -68,6 +72,8 @@ define amdgpu_kernel void @test_get_ddid(ptr addrspace(1) %out) {
;
; GFX1250-SDAG-LABEL: test_get_ddid:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DDID)
@@ -78,6 +84,8 @@ define amdgpu_kernel void @test_get_ddid(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-LABEL: test_get_ddid:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_GET_DDID)
@@ -113,6 +121,8 @@ define amdgpu_kernel void @test_get_tma(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: test_get_tma:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TMA)
@@ -149,6 +159,8 @@ define amdgpu_kernel void @test_get_realtime(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: test_get_realtime:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_REALTIME)
@@ -183,6 +195,8 @@ define amdgpu_kernel void @test_savewave(ptr addrspace(1) %out) {
;
; GFX1250-SDAG-LABEL: test_savewave:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_SAVE_WAVE)
@@ -193,6 +207,8 @@ define amdgpu_kernel void @test_savewave(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-LABEL: test_savewave:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(MSG_RTN_SAVE_WAVE)
@@ -228,6 +244,8 @@ define amdgpu_kernel void @test_get_tba(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: test_get_tba:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_TBA)
@@ -262,6 +280,8 @@ define amdgpu_kernel void @test_get_0_i32(ptr addrspace(1) %out) {
;
; GFX1250-SDAG-LABEL: test_get_0_i32:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(0, 0, 0)
@@ -272,6 +292,8 @@ define amdgpu_kernel void @test_get_0_i32(ptr addrspace(1) %out) {
;
; GFX1250-GISEL-LABEL: test_get_0_i32:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_sendmsg_rtn_b32 s2, sendmsg(0, 0, 0)
@@ -307,6 +329,8 @@ define amdgpu_kernel void @test_get_99999_i64(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: test_get_99999_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_sendmsg_rtn_b64 s[2:3], 99999
@@ -343,6 +367,8 @@ define amdgpu_kernel void @test_get_136_i64(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: test_get_136_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: s_sendmsg_rtn_b64 s[2:3], sendmsg(MSG_RTN_GET_CLUSTER_BARRIER_STATE)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll
index a5a2799789dc0..07c6f6c851d06 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.amdgcn.sin.bf16(bfloat) #0
define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-LABEL: sin_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -21,6 +23,8 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; REAL16-LABEL: sin_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -36,6 +40,8 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @sin_bf16_constant_4(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: sin_bf16_constant_4:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_sin_bf16_e32 v0, 4.0
@@ -46,6 +52,8 @@ define amdgpu_kernel void @sin_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: sin_bf16_constant_4:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_sin_bf16_e32 v0.l, 4.0
@@ -61,6 +69,8 @@ define amdgpu_kernel void @sin_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @sin_bf16_constant_100(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: sin_bf16_constant_100:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_sin_bf16_e32 v0, 0x42c8
@@ -71,6 +81,8 @@ define amdgpu_kernel void @sin_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: sin_bf16_constant_100:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_sin_bf16_e32 v0.l, 0x42c8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
index 04bda9c8595ca..114e2d8dd6586 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.amdgcn.sqrt.bf16(bfloat) #0
define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-LABEL: sqrt_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -21,6 +23,8 @@ define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; REAL16-LABEL: sqrt_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -36,6 +40,8 @@ define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @sqrt_bf16_constant_4(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: sqrt_bf16_constant_4:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_sqrt_bf16_e32 v0, 4.0
@@ -46,6 +52,8 @@ define amdgpu_kernel void @sqrt_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: sqrt_bf16_constant_4:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_sqrt_bf16_e32 v0.l, 4.0
@@ -61,6 +69,8 @@ define amdgpu_kernel void @sqrt_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @sqrt_bf16_constant_100(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: sqrt_bf16_constant_100:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_sqrt_bf16_e32 v0, 0x42c8
@@ -71,6 +81,8 @@ define amdgpu_kernel void @sqrt_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: sqrt_bf16_constant_100:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_sqrt_bf16_e32 v0.l, 0x42c8
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
index 807c1c962496d..736fc5bbdef91 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
@@ -34,6 +34,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32(<4 x i32> %addr, i32 %i
;
; GFX12-LABEL: struct_atomic_buffer_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -86,6 +88,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_const_idx(<4 x i32> %ad
;
; GFX12-LABEL: struct_atomic_buffer_load_i32_const_idx:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -139,6 +143,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_off(<4 x i32> %addr, i3
;
; GFX12-LABEL: struct_atomic_buffer_load_i32_off:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -194,6 +200,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_soff(<4 x i32> %addr, i
;
; GFX12-LABEL: struct_atomic_buffer_load_i32_soff:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -249,6 +257,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_dlc(<4 x i32> %addr, i3
;
; GFX12-LABEL: struct_atomic_buffer_load_i32_dlc:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -304,6 +314,8 @@ define amdgpu_kernel void @struct_nonatomic_buffer_load_i32(<4 x i32> %addr, i32
;
; GFX12-LABEL: struct_nonatomic_buffer_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -360,6 +372,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
;
; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_i64:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -384,6 +398,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
;
; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_i64:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -408,6 +424,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
;
; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_i64:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -432,6 +450,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
;
; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_i64:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -489,6 +509,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v2i16(<4 x i32> %addr, i32
;
; GFX12-LABEL: struct_atomic_buffer_load_v2i16:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -642,6 +664,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
;
; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -666,6 +690,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
;
; GFX12-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -692,6 +718,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
;
; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -718,6 +746,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
;
; GFX12-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -778,6 +808,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i32(<4 x i32> %addr, i32
;
; GFX12-LABEL: struct_atomic_buffer_load_v4i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -836,6 +868,8 @@ define amdgpu_kernel void @struct_atomic_buffer_load_ptr(<4 x i32> %addr, i32 %i
;
; GFX12-LABEL: struct_atomic_buffer_load_ptr:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll
index ef125398c1594..83652ccb02481 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll
@@ -67,6 +67,8 @@ define amdgpu_ps void @struct_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addr
;
; GFX1250-LABEL: struct_buffer_load_i8_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -143,6 +145,8 @@ define amdgpu_ps void @struct_buffer_load_i16_tfe(<4 x i32> inreg %rsrc, ptr add
;
; GFX1250-LABEL: struct_buffer_load_i16_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -219,6 +223,8 @@ define amdgpu_ps void @struct_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr add
;
; GFX1250-LABEL: struct_buffer_load_f16_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -295,6 +301,8 @@ define amdgpu_ps void @struct_buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr add
;
; GFX1250-LABEL: struct_buffer_load_i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -391,6 +399,8 @@ define amdgpu_ps void @struct_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr a
;
; GFX1250-LABEL: struct_buffer_load_v2i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -487,6 +497,8 @@ define amdgpu_ps void @struct_buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr a
;
; GFX1250-LABEL: struct_buffer_load_v2f32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -589,6 +601,8 @@ define amdgpu_ps void @struct_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr a
;
; GFX1250-LABEL: struct_buffer_load_v3i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -692,6 +706,8 @@ define amdgpu_ps void @struct_buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr a
;
; GFX1250-LABEL: struct_buffer_load_v3f32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -782,6 +798,8 @@ define amdgpu_ps void @struct_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr a
;
; GFX1250-LABEL: struct_buffer_load_v4i32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -872,6 +890,8 @@ define amdgpu_ps void @struct_buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr a
;
; GFX1250-LABEL: struct_buffer_load_v4f32_tfe:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
index 573f4645c50e4..476ea82afc372 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
@@ -26,6 +26,8 @@ define amdgpu_ps void @buffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <
;
; GFX12-LABEL: buffer_store:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: v_mov_b32_e32 v12, 0
; GFX12-NEXT: s_clause 0x2
@@ -55,6 +57,8 @@ define amdgpu_ps void @buffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
;
; GFX12-LABEL: buffer_store_immoffs:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: v_mov_b32_e32 v4, 0
; GFX12-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null idxen offset:42
@@ -77,6 +81,8 @@ define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float>, i32) {
;
; GFX12-LABEL: buffer_store_idx:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
@@ -104,6 +110,8 @@ define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float>, i32) {
;
; GFX12-LABEL: buffer_store_ofs:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, 0
; GFX12-NEXT: buffer_store_b128 v[0:3], v[4:5], s[0:3], null idxen offen
@@ -126,6 +134,8 @@ define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float>, i32, i32)
;
; GFX12-LABEL: buffer_store_both:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b128 v[0:3], v[4:5], s[0:3], null idxen offen
; GFX12-NEXT: s_endpgm
@@ -149,6 +159,8 @@ define amdgpu_ps void @buffer_store_both_reversed(<4 x i32> inreg, <4 x float>,
;
; GFX12-LABEL: buffer_store_both_reversed:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: v_dual_mov_b32 v6, v5 :: v_dual_mov_b32 v7, v4
; GFX12-NEXT: buffer_store_b128 v[0:3], v[6:7], s[0:3], null idxen offen
@@ -187,6 +199,8 @@ define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float>, i32, i32,
;
; GFX12-LABEL: buffer_store_wait:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: buffer_store_b128 v[0:3], v4, s[0:3], null idxen
@@ -214,6 +228,8 @@ define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %
;
; GFX12-LABEL: buffer_store_x1:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b32 v0, v1, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
@@ -235,6 +251,8 @@ define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data,
;
; GFX12-LABEL: buffer_store_x2:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
@@ -263,6 +281,8 @@ define amdgpu_ps void @buffer_store_int(<4 x i32> inreg, <4 x i32>, <2 x i32>, i
;
; GFX12-LABEL: buffer_store_int:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: v_mov_b32_e32 v7, 0
; GFX12-NEXT: s_clause 0x2
@@ -292,6 +312,8 @@ define amdgpu_ps void @struct_buffer_store_byte(<4 x i32> inreg %rsrc, float %v1
;
; GFX12-LABEL: struct_buffer_store_byte:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX12-NEXT: buffer_store_b8 v0, v1, s[0:3], null idxen
@@ -324,6 +346,8 @@ define amdgpu_ps void @struct_buffer_store_f16(<4 x i32> inreg %rsrc, float %v1,
;
; GFX12-TRUE16-LABEL: struct_buffer_store_f16:
; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: global_wb
+; GFX12-TRUE16-NEXT: v_nop
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
@@ -332,6 +356,8 @@ define amdgpu_ps void @struct_buffer_store_f16(<4 x i32> inreg %rsrc, float %v1,
;
; GFX12-FAKE16-LABEL: struct_buffer_store_f16:
; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: global_wb
+; GFX12-FAKE16-NEXT: v_nop
; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
@@ -355,6 +381,8 @@ define amdgpu_ps void @struct_buffer_store_v2f16(<4 x i32> inreg %rsrc, <2 x hal
;
; GFX12-LABEL: struct_buffer_store_v2f16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b32 v0, v1, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
@@ -375,6 +403,8 @@ define amdgpu_ps void @struct_buffer_store_v4f16(<4 x i32> inreg %rsrc, <4 x hal
;
; GFX12-LABEL: struct_buffer_store_v4f16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
@@ -397,6 +427,8 @@ define amdgpu_ps void @struct_buffer_store_i16(<4 x i32> inreg %rsrc, float %v1,
;
; GFX12-LABEL: struct_buffer_store_i16:
; GFX12: ; %bb.0: ; %main_body
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX12-NEXT: buffer_store_b16 v0, v1, s[0:3], null idxen
@@ -421,6 +453,8 @@ define amdgpu_ps void @struct_buffer_store_vif16(<4 x i32> inreg %rsrc, <2 x i16
;
; GFX12-LABEL: struct_buffer_store_vif16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b32 v0, v1, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
@@ -441,6 +475,8 @@ define amdgpu_ps void @struct_buffer_store_v4i16(<4 x i32> inreg %rsrc, <4 x i16
;
; GFX12-LABEL: struct_buffer_store_v4i16:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: buffer_store_b64 v[0:1], v2, s[0:3], null idxen
; GFX12-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
index 1744202881d00..d3ba99275f7f6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
@@ -34,6 +34,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32(ptr addrspace(8) %p
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -86,6 +88,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_const_idx(ptr addrs
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -139,6 +143,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_off(ptr addrspace(8
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_off:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -194,6 +200,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_soff(ptr addrspace(
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -249,6 +257,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -304,6 +314,8 @@ define amdgpu_kernel void @struct_ptr_nonatomic_buffer_load_i32(ptr addrspace(8)
;
; GFX12-LABEL: struct_ptr_nonatomic_buffer_load_i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -360,6 +372,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
;
; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -384,6 +398,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
;
; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -408,6 +424,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
;
; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_i64:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -432,6 +450,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
;
; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_i64:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -489,6 +509,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v2i16(ptr addrspace(8)
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_v2i16:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -642,6 +664,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
;
; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT: global_wb
+; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -666,6 +690,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
;
; GFX12-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %bb
+; GFX12-SDAG-FAKE16-NEXT: global_wb
+; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -692,6 +718,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
;
; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT: global_wb
+; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -718,6 +746,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
;
; GFX12-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX12-GISEL-FAKE16: ; %bb.0: ; %bb
+; GFX12-GISEL-FAKE16-NEXT: global_wb
+; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -778,6 +808,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i32(ptr addrspace(8)
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_v4i32:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
@@ -836,6 +868,8 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %p
;
; GFX12-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX12: ; %bb.0: ; %bb
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll
index 184a89e8ce854..766ab93e26214 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll
@@ -99,6 +99,8 @@ define float @tanh_f32_neg_abs(float %src) {
define amdgpu_ps float @tanh_f32_scalar(float inreg %src) {
; GFX1250-LABEL: tanh_f32_scalar:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_tanh_f32_e32 v0, s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -115,6 +117,8 @@ define amdgpu_ps float @tanh_f32_scalar(float inreg %src) {
define amdgpu_ps float @tanh_f32_constant_4.0() {
; GFX1250-LABEL: tanh_f32_constant_4.0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_tanh_f32_e32 v0, 4.0
; GFX1250-NEXT: ; return to shader part epilog
@@ -130,6 +134,8 @@ define amdgpu_ps float @tanh_f32_constant_4.0() {
define amdgpu_ps float @tanh_f32_constant_100.0() {
; GFX1250-LABEL: tanh_f32_constant_100.0:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_tanh_f32_e32 v0, 0x42c80000
; GFX1250-NEXT: ; return to shader part epilog
@@ -145,6 +151,8 @@ define amdgpu_ps float @tanh_f32_constant_100.0() {
define amdgpu_ps float @tanh_undef_f32() {
; GFX1250-LABEL: tanh_undef_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: ; return to shader part epilog
;
@@ -314,12 +322,16 @@ define half @tanh_f16_neg_abs(half %src) {
define amdgpu_ps half @tanh_f16_scalar(half inreg %src) {
; GFX1250-SDAG-REAL16-LABEL: tanh_f16_scalar:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_tanh_f16_e32 v0.l, s0
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: tanh_f16_scalar:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_tanh_f16_e32 v0, s0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -340,12 +352,16 @@ define amdgpu_ps half @tanh_f16_scalar(half inreg %src) {
define amdgpu_ps half @tanh_f16_constant_4.0() {
; GFX1250-SDAG-REAL16-LABEL: tanh_f16_constant_4.0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_tanh_f16_e32 v0.l, 4.0
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: tanh_f16_constant_4.0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_tanh_f16_e32 v0, 4.0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -366,12 +382,16 @@ define amdgpu_ps half @tanh_f16_constant_4.0() {
define amdgpu_ps half @tanh_f16_constant_100.0() {
; GFX1250-SDAG-REAL16-LABEL: tanh_f16_constant_100.0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_tanh_f16_e32 v0.l, 0x5640
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: tanh_f16_constant_100.0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_tanh_f16_e32 v0, 0x5640
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -392,6 +412,8 @@ define amdgpu_ps half @tanh_f16_constant_100.0() {
define amdgpu_ps half @tanh_undef_f16() {
; GFX1250-LABEL: tanh_undef_f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: ; return to shader part epilog
;
@@ -561,12 +583,16 @@ define bfloat @tanh_bf16_neg_abs(bfloat %src) {
define amdgpu_ps bfloat @tanh_bf16_scalar(bfloat inreg %src) {
; GFX1250-SDAG-REAL16-LABEL: tanh_bf16_scalar:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_tanh_bf16_e32 v0.l, s0
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: tanh_bf16_scalar:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_tanh_bf16_e32 v0, s0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -587,12 +613,16 @@ define amdgpu_ps bfloat @tanh_bf16_scalar(bfloat inreg %src) {
define amdgpu_ps bfloat @tanh_bf16_constant_4.0() {
; GFX1250-SDAG-REAL16-LABEL: tanh_bf16_constant_4.0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_tanh_bf16_e32 v0.l, 4.0
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: tanh_bf16_constant_4.0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_tanh_bf16_e32 v0, 4.0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -613,12 +643,16 @@ define amdgpu_ps bfloat @tanh_bf16_constant_4.0() {
define amdgpu_ps bfloat @tanh_bf16_constant_100.0() {
; GFX1250-SDAG-REAL16-LABEL: tanh_bf16_constant_100.0:
; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: global_wb
+; GFX1250-SDAG-REAL16-NEXT: v_nop
; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-REAL16-NEXT: v_tanh_bf16_e32 v0.l, 0x42c8
; GFX1250-SDAG-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-FAKE16-LABEL: tanh_bf16_constant_100.0:
; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: global_wb
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_tanh_bf16_e32 v0, 0x42c8
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -639,6 +673,8 @@ define amdgpu_ps bfloat @tanh_bf16_constant_100.0() {
define amdgpu_ps bfloat @tanh_undef_bf16() {
; GFX1250-LABEL: tanh_undef_bf16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
index 0ae3784e87361..7e9af706c4b01 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
@@ -12,6 +12,8 @@ declare void @llvm.amdgcn.global.load.async.to.lds.b32(ptr addrspace(1) %src, pt
define amdgpu_ps void @tensor_load_to_lds_d4(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
; GFX1250-LABEL: tensor_load_to_lds_d4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19]
; GFX1250-NEXT: s_endpgm
@@ -22,6 +24,8 @@ define amdgpu_ps void @tensor_load_to_lds_d4(<4 x i32> inreg %D0, <8 x i32> inre
define amdgpu_ps void @tensor_load_to_lds_d4_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
; GFX1250-SDAG-LABEL: tensor_load_to_lds_d4_vector:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
@@ -49,6 +53,8 @@ define amdgpu_ps void @tensor_load_to_lds_d4_vector(<4 x i32> %D0, <8 x i32> %D1
;
; GFX1250-GISEL-LABEL: tensor_load_to_lds_d4_vector:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
@@ -80,6 +86,8 @@ define amdgpu_ps void @tensor_load_to_lds_d4_vector(<4 x i32> %D0, <8 x i32> %D1
define amdgpu_ps void @tensor_load_to_lds_d2(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
; GFX1250-LABEL: tensor_load_to_lds_d2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
@@ -90,6 +98,8 @@ define amdgpu_ps void @tensor_load_to_lds_d2(<4 x i32> inreg %D0, <8 x i32> inre
define amdgpu_ps void @tensor_load_to_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
; GFX1250-SDAG-LABEL: tensor_load_to_lds_d2_vector:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s8, v0
@@ -109,6 +119,8 @@ define amdgpu_ps void @tensor_load_to_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1
;
; GFX1250-GISEL-LABEL: tensor_load_to_lds_d2_vector:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
@@ -132,6 +144,8 @@ define amdgpu_ps void @tensor_load_to_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1
define amdgpu_ps void @tensor_store_from_lds_d4(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
; GFX1250-LABEL: tensor_store_from_lds_d4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
@@ -142,6 +156,8 @@ define amdgpu_ps void @tensor_store_from_lds_d4(<4 x i32> inreg %D0, <8 x i32> i
define amdgpu_ps void @tensor_store_from_lds_d4_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
; GFX1250-SDAG-LABEL: tensor_store_from_lds_d4_vector:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
@@ -169,6 +185,8 @@ define amdgpu_ps void @tensor_store_from_lds_d4_vector(<4 x i32> %D0, <8 x i32>
;
; GFX1250-GISEL-LABEL: tensor_store_from_lds_d4_vector:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
@@ -200,6 +218,8 @@ define amdgpu_ps void @tensor_store_from_lds_d4_vector(<4 x i32> %D0, <8 x i32>
define amdgpu_ps void @tensor_store_from_lds_d2(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
; GFX1250-LABEL: tensor_store_from_lds_d2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_store_from_lds s[0:3], s[4:11]
; GFX1250-NEXT: s_endpgm
@@ -210,6 +230,8 @@ define amdgpu_ps void @tensor_store_from_lds_d2(<4 x i32> inreg %D0, <8 x i32> i
define amdgpu_ps void @tensor_store_from_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
; GFX1250-SDAG-LABEL: tensor_store_from_lds_d2_vector:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s8, v0
@@ -229,6 +251,8 @@ define amdgpu_ps void @tensor_store_from_lds_d2_vector(<4 x i32> %D0, <8 x i32>
;
; GFX1250-GISEL-LABEL: tensor_store_from_lds_d2_vector:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s8, v0
; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s9, v1
@@ -258,6 +282,8 @@ define amdgpu_ps void @tensor_store_from_lds_d2_vector(<4 x i32> %D0, <8 x i32>
define amdgpu_ps void @tensor_load_to_lds_d5(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3, <8 x i32> inreg %D4) {
; GFX1250-LABEL: tensor_load_to_lds_d5:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19]
; GFX1250-NEXT: s_endpgm
@@ -268,6 +294,8 @@ define amdgpu_ps void @tensor_load_to_lds_d5(<4 x i32> inreg %D0, <8 x i32> inre
define amdgpu_ps void @tensor_store_from_lds_d5(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3, <8 x i32> inreg %D4) {
; GFX1250-LABEL: tensor_store_from_lds_d5:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
@@ -284,6 +312,8 @@ define amdgpu_ps void @tensor_store_from_lds_d5(<4 x i32> inreg %D0, <8 x i32> i
define amdgpu_ps void @tensor_load_to_lds_with_asyncmark(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
; GFX1250-LABEL: tensor_load_to_lds_with_asyncmark:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
; GFX1250-NEXT: ; asyncmark
@@ -299,6 +329,8 @@ define amdgpu_ps void @tensor_load_to_lds_with_asyncmark(<4 x i32> inreg %D0, <8
define amdgpu_ps void @tensor_store_from_lds_with_asyncmark(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
; GFX1250-LABEL: tensor_store_from_lds_with_asyncmark:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_store_from_lds s[0:3], s[4:11]
; GFX1250-NEXT: ; asyncmark
@@ -317,6 +349,8 @@ define amdgpu_ps void @tensor_store_from_lds_with_asyncmark(<4 x i32> inreg %D0,
define amdgpu_ps void @tensor_load_to_lds_two_asyncmarks(<4 x i32> inreg %D0a, <8 x i32> inreg %D1a, <4 x i32> inreg %D0b, <8 x i32> inreg %D1b, ptr addrspace(3) %lds) {
; GFX1250-LABEL: tensor_load_to_lds_two_asyncmarks:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: tensor_load_to_lds s[0:3], s[4:11]
; GFX1250-NEXT: ; asyncmark
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
index 0541222d2b255..12475c81adf9e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
@@ -89,6 +89,8 @@ define amdgpu_kernel void @dpp_test(ptr addrspace(1) %out, i32 %in1, i32 %in2) {
;
; GFX1250-LABEL: dpp_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -188,6 +190,8 @@ define amdgpu_kernel void @dpp_test_bc(ptr addrspace(1) %out, i32 %in1, i32 %in2
;
; GFX1250-LABEL: dpp_test_bc:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -322,6 +326,8 @@ define weak_odr amdgpu_kernel void @dpp_test1(ptr %arg) local_unnamed_addr {
;
; GFX1250-LABEL: dpp_test1:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
@@ -465,6 +471,8 @@ define amdgpu_kernel void @update_dppi64_test(ptr addrspace(1) %arg, i64 %in1, i
;
; GFX1250-LABEL: update_dppi64_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -596,6 +604,8 @@ define amdgpu_kernel void @update_dppf64_test(ptr addrspace(1) %arg, double %in1
;
; GFX1250-LABEL: update_dppf64_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -727,6 +737,8 @@ define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32>
;
; GFX1250-LABEL: update_dppv2i32_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -858,6 +870,8 @@ define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x floa
;
; GFX1250-LABEL: update_dppv2f32_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -989,6 +1003,8 @@ define amdgpu_kernel void @update_dpp_p0_test(ptr addrspace(1) %arg, ptr %in1, p
;
; GFX1250-LABEL: update_dpp_p0_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1092,6 +1108,8 @@ define amdgpu_kernel void @update_dpp_p3_test(ptr addrspace(3) %arg, ptr addrspa
;
; GFX1250-LABEL: update_dpp_p3_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1209,6 +1227,8 @@ define amdgpu_kernel void @update_dpp_p5_test(ptr addrspace(5) %arg, ptr addrspa
;
; GFX1250-LABEL: update_dpp_p5_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1337,6 +1357,8 @@ define amdgpu_kernel void @update_dppi64_imm_old_test(ptr addrspace(1) %arg, i64
;
; GFX1250-LABEL: update_dppi64_imm_old_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1466,6 +1488,8 @@ define amdgpu_kernel void @update_dppf64_imm_old_test(ptr addrspace(1) %arg, dou
;
; GFX1250-LABEL: update_dppf64_imm_old_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1592,6 +1616,8 @@ define amdgpu_kernel void @update_dppi64_imm_src_test(ptr addrspace(1) %out, i64
;
; GFX1250-LABEL: update_dppi64_imm_src_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0x7047
@@ -1717,6 +1743,8 @@ define amdgpu_kernel void @update_dppf64_imm_src_test(ptr addrspace(1) %out, dou
;
; GFX1250-LABEL: update_dppf64_imm_src_test:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0x405edce1
@@ -1819,6 +1847,8 @@ define amdgpu_kernel void @dpp_test_f32(ptr addrspace(1) %out, float %in1, float
;
; GFX1250-LABEL: dpp_test_f32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1918,6 +1948,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb1(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2017,6 +2049,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb2(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2116,6 +2150,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb3(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2215,6 +2251,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb4(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2314,6 +2352,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb5(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb5:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2413,6 +2453,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb6(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb6:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2513,6 +2555,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb7(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb7:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2612,6 +2656,8 @@ define amdgpu_kernel void @dpp_test_f32_imm_comb8(ptr addrspace(1) %out, float %
;
; GFX1250-LABEL: dpp_test_f32_imm_comb8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2711,6 +2757,8 @@ define amdgpu_kernel void @dpp_test_v2i16(ptr addrspace(1) %out, <2 x i16> %in1,
;
; GFX1250-LABEL: dpp_test_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2810,6 +2858,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb1(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2909,6 +2959,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb2(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3008,6 +3060,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb3(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3107,6 +3161,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb4(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3206,6 +3262,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb5(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb5:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3305,6 +3363,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb6(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb6:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3404,6 +3464,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb7(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb7:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3503,6 +3565,8 @@ define amdgpu_kernel void @dpp_test_v2i16_imm_comb8(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2i16_imm_comb8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3602,6 +3666,8 @@ define amdgpu_kernel void @dpp_test_v2f16(ptr addrspace(1) %out, <2 x half> %in1
;
; GFX1250-LABEL: dpp_test_v2f16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3701,6 +3767,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb1(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3800,6 +3868,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb2(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb2:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3899,6 +3969,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb3(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb3:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3998,6 +4070,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb4(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb4:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4097,6 +4171,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb5(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb5:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4196,6 +4272,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb6(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb6:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4295,6 +4373,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb7(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb7:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4394,6 +4474,8 @@ define amdgpu_kernel void @dpp_test_v2f16_imm_comb8(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: dpp_test_v2f16_imm_comb8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
index 6c414c1989ce1..eea194ff23690 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
@@ -29,6 +29,8 @@ define amdgpu_cs void @test_wave_id(ptr addrspace(1) %out) {
;
; GFX1250-LABEL: test_wave_id:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_bfe_u32 s0, ttmp8, 0x50019
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1250.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1250.w32.ll
index 5af8df6fce69b..ee6fa1a8b176c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1250.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1250.w32.ll
@@ -7,6 +7,8 @@
define amdgpu_ps void @test_wmma_f32_16x16x4_f32(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x4_f32:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -16,6 +18,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x4_f32(<2 x float> %A, <2 x float> %B,
;
; GISEL-LABEL: test_wmma_f32_16x16x4_f32:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -31,6 +35,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -40,6 +46,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloa
;
; GISEL-LABEL: test_wmma_f32_16x16x32_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -55,6 +63,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -62,6 +72,8 @@ define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16(<16 x bfloat> %A, <16 x bflo
;
; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -75,6 +87,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -82,6 +96,8 @@ define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16(<16 x bfloat> %A, <16 x b
;
; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -95,6 +111,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -104,6 +122,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -119,6 +139,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -128,6 +150,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -143,6 +167,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -152,6 +178,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -167,6 +195,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -176,6 +206,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -191,6 +223,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -198,6 +232,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -211,6 +247,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -218,6 +256,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -231,6 +271,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -238,6 +280,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -251,6 +295,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -258,6 +304,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -271,6 +319,8 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x64_iu8(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -280,6 +330,8 @@ define amdgpu_ps void @test_wmma_i32_16x16x64_iu8(<8 x i32> %A, <8 x i32> %B, <8
;
; GISEL-LABEL: test_wmma_i32_16x16x64_iu8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -295,6 +347,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_f16(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -304,6 +358,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_f16(<16 x half> %A, <16 x half> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x32_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -319,6 +375,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x32_f16(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x32_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -326,6 +384,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x32_f16(<16 x half> %A, <16 x half> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x32_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -339,6 +399,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -348,6 +410,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -363,6 +427,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -372,6 +438,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -387,6 +455,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -396,6 +466,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -411,6 +483,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -420,6 +494,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_b_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -435,6 +511,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -444,6 +522,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp8_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -459,6 +539,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -468,6 +550,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -483,6 +567,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -492,6 +578,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -507,6 +595,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -516,6 +606,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -531,6 +623,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -540,6 +634,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -555,6 +651,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -564,6 +662,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf8_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -579,6 +679,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -588,6 +690,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -603,6 +707,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -612,6 +718,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -627,6 +735,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -636,6 +746,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -651,6 +763,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -660,6 +774,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -675,6 +791,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -684,6 +802,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp6_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -699,6 +819,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -708,6 +830,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -723,6 +847,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -732,6 +858,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -747,6 +875,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -756,6 +886,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -771,6 +903,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -780,6 +914,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -795,6 +931,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -804,6 +942,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_bf6_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -819,6 +959,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -828,6 +970,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -843,6 +987,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -852,6 +998,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -867,6 +1015,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -876,6 +1026,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -891,6 +1043,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -900,6 +1054,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A, <12
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -915,6 +1071,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -924,6 +1082,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_fp4_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -939,6 +1099,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x1
@@ -948,6 +1110,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x1
@@ -963,6 +1127,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_ss(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ss:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s0, s1 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -972,6 +1138,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_ss(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ss:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s0, s1 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_reuse
; GISEL-NEXT: s_clause 0x1
@@ -987,6 +1155,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_si_scale(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 inreg %scale_src0, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_si_scale:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v42, 0x64
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -998,6 +1168,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_si_scale(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_si_scale:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b32_e32 v42, 0x64
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1015,6 +1187,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1024,6 +1198,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1039,6 +1215,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1048,6 +1226,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1063,6 +1243,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -1072,6 +1254,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_b_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -1087,6 +1271,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1096,6 +1282,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp8_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1111,6 +1299,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1120,6 +1310,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1135,6 +1327,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1144,6 +1338,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v40, v41 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1159,6 +1355,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1168,6 +1366,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1183,6 +1383,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -1192,6 +1394,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -1207,6 +1411,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1216,6 +1422,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf8_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1231,6 +1439,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1240,6 +1450,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1255,6 +1467,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1264,6 +1478,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1279,6 +1495,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1288,6 +1506,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1303,6 +1523,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1312,6 +1534,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1327,6 +1551,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1336,6 +1562,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp6_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1351,6 +1579,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -1360,6 +1590,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -1375,6 +1607,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1384,6 +1618,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v36, v37 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1399,6 +1635,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1408,6 +1646,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1423,6 +1663,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1432,6 +1674,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1447,6 +1691,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1456,6 +1702,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_bf6_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1471,6 +1719,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1480,6 +1730,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1495,6 +1747,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1504,6 +1758,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v32, v33 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1519,6 +1775,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1528,6 +1786,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1543,6 +1803,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1552,6 +1814,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v28, v29 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1567,6 +1831,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v24, v25 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1576,6 +1842,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_fp4_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v24, v25 matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1591,6 +1859,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x1
@@ -1600,6 +1870,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4(<16 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x1
@@ -1615,6 +1887,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_ss(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ss:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s[0:1], s[2:3] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -1624,6 +1898,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_ss(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ss:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], s[0:1], s[2:3] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GISEL-NEXT: s_clause 0x1
@@ -1639,6 +1915,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_si_scale(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 inreg %scale_src0, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_si_scale:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[42:43], 0x64
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1650,6 +1928,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_si_scale(<16 x i32
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_si_scale:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b64_e32 v[42:43], 0x64
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1667,6 +1947,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1676,6 +1958,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf8(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1691,6 +1975,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1700,6 +1986,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp6(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1715,6 +2003,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -1724,6 +2014,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf6(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_b_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -1739,6 +2031,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1748,6 +2042,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp4(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp8_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1763,6 +2059,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1772,6 +2070,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp8(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1787,6 +2087,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1796,6 +2098,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf8(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], v[40:41], v[42:43] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1811,6 +2115,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1820,6 +2126,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp6(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1835,6 +2143,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -1844,6 +2154,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf6(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:15], v[16:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -1859,6 +2171,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1868,6 +2182,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp4(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf8_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:15], v[16:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF8 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1883,6 +2199,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1892,6 +2210,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp8(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1907,6 +2227,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -1916,6 +2238,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf8(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -1931,6 +2255,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -1940,6 +2266,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp6(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -1955,6 +2283,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1964,6 +2294,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf6(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -1979,6 +2311,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -1988,6 +2322,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp4(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp6_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -2003,6 +2339,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF6
; GFX1250-NEXT: s_clause 0x1
@@ -2012,6 +2350,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp8(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF6
; GISEL-NEXT: s_clause 0x1
@@ -2027,6 +2367,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -2036,6 +2378,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf8(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[28:35], v[0:11], v[12:27], v[28:35], v[36:37], v[38:39] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -2051,6 +2395,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -2060,6 +2406,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp6(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -2075,6 +2423,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -2084,6 +2434,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf6(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:11], v[12:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -2099,6 +2451,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -2108,6 +2462,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp4(<12 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_bf6_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:11], v[12:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_BF6 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -2123,6 +2479,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -2132,6 +2490,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp8(<8 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -2147,6 +2507,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8
; GFX1250-NEXT: s_clause 0x1
@@ -2156,6 +2518,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf8(<8 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[24:31], v[0:7], v[8:23], v[24:31], v[32:33], v[34:35] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_BF8
; GISEL-NEXT: s_clause 0x1
@@ -2171,6 +2535,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6
; GFX1250-NEXT: s_clause 0x1
@@ -2180,6 +2546,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp6(<8 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP6
; GISEL-NEXT: s_clause 0x1
@@ -2195,6 +2563,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32> %A, <12 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf6:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -2204,6 +2574,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf6(<8 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_bf6:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[20:27], v[0:7], v[8:19], v[20:27], v[28:29], v[30:31] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -2219,6 +2591,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v[24:25], v[26:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GFX1250-NEXT: s_clause 0x1
@@ -2228,6 +2602,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp4(<8 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_fp4_fp4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[16:23], v[0:7], v[8:15], v[16:23], v[24:25], v[26:27] matrix_a_fmt:MATRIX_FMT_FP4 matrix_b_fmt:MATRIX_FMT_FP4
; GISEL-NEXT: s_clause 0x1
@@ -2243,6 +2619,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2250,6 +2628,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2263,6 +2643,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2270,6 +2652,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2283,6 +2667,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2290,6 +2676,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2303,6 +2691,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2310,6 +2700,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] matrix_b_reuse
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -2323,6 +2715,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2332,6 +2726,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2347,6 +2743,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2356,6 +2754,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2371,6 +2771,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2380,6 +2782,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2395,6 +2799,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2404,6 +2810,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2419,6 +2827,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_32x16x128_f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]
; GFX1250-NEXT: s_clause 0x3
@@ -2430,6 +2840,8 @@ define amdgpu_ps void @test_wmma_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, <
;
; GISEL-LABEL: test_wmma_f32_32x16x128_f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]
; GISEL-NEXT: s_clause 0x3
@@ -2447,6 +2859,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i32 %scale_src0, i32 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v40, v41 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x3
@@ -2458,6 +2872,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4(<16 x i32> %A, <8 x i32>
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v40, v41 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x3
@@ -2475,6 +2891,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_ss(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_ss:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s0, s1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x3
@@ -2486,6 +2904,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_ss(<16 x i32> %A, <8 x i
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_ss:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s0, s1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse
; GISEL-NEXT: s_clause 0x3
@@ -2503,6 +2923,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_si_scale(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i32 inreg %scale_src0, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_si_scale:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v42, 0x64
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2516,6 +2938,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_si_scale(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_si_scale:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b32_e32 v42, 0x64
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2535,6 +2959,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i64 %scale_src0, i64 %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v[40:41], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x3
@@ -2546,6 +2972,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4(<16 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], v[40:41], v[42:43] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x3
@@ -2563,6 +2991,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_ss(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_ss:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s[0:1], s[2:3] matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x3
@@ -2574,6 +3004,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_ss(<16 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_ss:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], s[0:1], s[2:3] matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_scale_fmt:MATRIX_SCALE_FMT_E5M3 matrix_b_scale_fmt:MATRIX_SCALE_FMT_E4M3 matrix_a_reuse
; GISEL-NEXT: s_clause 0x3
@@ -2591,6 +3023,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_si_scale(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, i64 inreg %scale_src0, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_si_scale:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[42:43], 0x64
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2604,6 +3038,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_si_scale(<16 x i32> %A
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_si_scale:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b64_e32 v[42:43], 0x64
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2623,6 +3059,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2633,6 +3071,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfl
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2649,6 +3089,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse
; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2657,6 +3099,8 @@ define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16(<16 x bfloat> %A, <32 x bf
;
; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2671,6 +3115,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2681,6 +3127,8 @@ define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16(<16 x bfloat> %A, <32 x
;
; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2697,6 +3145,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2706,6 +3156,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2721,6 +3173,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2730,6 +3184,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2745,6 +3201,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2754,6 +3212,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2769,6 +3229,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2778,6 +3240,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2793,6 +3257,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-FAKE16-LABEL: test_swmmac_f16_16x16x128_fp8_fp8:
; GFX1250-FAKE16: ; %bb.0: ; %bb
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v29, 0
@@ -2803,6 +3269,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v30
@@ -2811,6 +3279,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8(<8 x i32> %A, <16 x i32
;
; GFX1250-REAL16-LABEL: test_swmmac_f16_16x16x128_fp8_fp8:
; GFX1250-REAL16: ; %bb.0: ; %bb
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2825,6 +3295,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-FAKE16-LABEL: test_swmmac_f16_16x16x128_fp8_bf8:
; GFX1250-FAKE16: ; %bb.0: ; %bb
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v29, 0
@@ -2835,6 +3307,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v30
@@ -2843,6 +3317,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8(<8 x i32> %A, <16 x i32
;
; GFX1250-REAL16-LABEL: test_swmmac_f16_16x16x128_fp8_bf8:
; GFX1250-REAL16: ; %bb.0: ; %bb
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2857,6 +3333,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-FAKE16-LABEL: test_swmmac_f16_16x16x128_bf8_fp8:
; GFX1250-FAKE16: ; %bb.0: ; %bb
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v29, 0
@@ -2867,6 +3345,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v30
@@ -2875,6 +3355,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8(<8 x i32> %A, <16 x i32
;
; GFX1250-REAL16-LABEL: test_swmmac_f16_16x16x128_bf8_fp8:
; GFX1250-REAL16: ; %bb.0: ; %bb
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2889,6 +3371,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-FAKE16-LABEL: test_swmmac_f16_16x16x128_bf8_bf8:
; GFX1250-FAKE16: ; %bb.0: ; %bb
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v29, 0
@@ -2899,6 +3383,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8(<8 x i32> %A, <16 x i32
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v30
@@ -2907,6 +3393,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8(<8 x i32> %A, <16 x i32
;
; GFX1250-REAL16-LABEL: test_swmmac_f16_16x16x128_bf8_bf8:
; GFX1250-REAL16: ; %bb.0: ; %bb
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] matrix_b_reuse
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2921,6 +3409,8 @@ bb:
define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, i64 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -2930,6 +3420,8 @@ define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8(<8 x i32> %A, <16 x i32> %B
;
; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] matrix_b_reuse
; GISEL-NEXT: s_clause 0x1
@@ -2945,6 +3437,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2955,6 +3449,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_f16(<16 x half> %A, <32 x half>
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v36, v33 :: v_dual_mov_b32 v37, v34
@@ -2971,6 +3467,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse
; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2979,6 +3477,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x64_f16(<16 x half> %A, <32 x half>
;
; GISEL-LABEL: test_swmmac_f16_16x16x64_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 matrix_b_reuse
; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v30
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll
index 9b4d2df8148fb..eb9a821c52c84 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_matrix_a_reuse(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_matrix_a_reuse:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_a_reuse
; GFX1251-NEXT: s_clause 0x3
@@ -16,6 +18,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_matrix_a_reuse(<2 x double> %A,
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_matrix_a_reuse:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_a_reuse
; GISEL-NEXT: s_clause 0x3
@@ -33,6 +37,8 @@ bb:
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_matrix_b_reuse(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_matrix_b_reuse:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_b_reuse
; GFX1251-NEXT: s_clause 0x3
@@ -44,6 +50,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_matrix_b_reuse(<2 x double> %A,
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_matrix_b_reuse:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] matrix_b_reuse
; GISEL-NEXT: s_clause 0x3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1250.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1250.w32.ll
index 2299ece622c24..20952f59d77a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1250.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1250.w32.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @test_wmma_f32_16x16x4_f32(<2 x float> %A, <2 x float> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x4_f32:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -14,6 +16,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x4_f32(<2 x float> %A, <2 x float> %B,
;
; GISEL-LABEL: test_wmma_f32_16x16x4_f32:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[6:13], v[0:1], v[2:3], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -29,6 +33,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x4_f32_non_splat(<2 x float> %A, <2 x float> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v6, 1.0 :: v_dual_mov_b32 v8, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -43,6 +49,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x4_f32_non_splat(<2 x float> %A, <2 x
;
; GISEL-LABEL: test_wmma_f32_16x16x4_f32_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -71,6 +79,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x4_f32_non_inlineable(<2 x float> %A, <2 x float> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v6, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -86,6 +96,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x4_f32_non_inlineable(<2 x float> %A,
;
; GISEL-LABEL: test_wmma_f32_16x16x4_f32_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -115,6 +127,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -124,6 +138,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloa
;
; GISEL-LABEL: test_wmma_f32_16x16x32_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[18:25], v[0:7], v[8:15], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -139,6 +155,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_non_splat(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -153,6 +171,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_non_splat(<16 x bfloat> %A, <
;
; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -173,6 +193,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_non_inlineable(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -188,6 +210,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_non_inlineable(<16 x bfloat>
;
; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b32_e32 v18, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -209,6 +233,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -216,6 +242,8 @@ define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16(<16 x bfloat> %A, <16 x bflo
;
; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0
; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -229,6 +257,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_non_splat(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 0x3f803f80 :: v_dual_mov_b32 v19, 1.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -239,6 +269,8 @@ define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_non_splat(<16 x bfloat> %A,
;
; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_dual_mov_b32 v18, 0x3f803f80 :: v_dual_mov_b32 v19, 1.0
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -255,6 +287,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_non_inlineable(<16 x bfloat> %A, <16 x bfloat> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3fc03fc0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -266,6 +300,8 @@ define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_non_inlineable(<16 x bfloat>
;
; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b32_e32 v18, 0x3fc03fc0
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -283,6 +319,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -290,6 +328,8 @@ define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16(<16 x bfloat> %A, <16 x b
;
; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[18:21], v[0:7], v[8:15], 1.0
; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -303,6 +343,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_non_splat(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -315,6 +357,8 @@ define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_non_splat(<16 x bfloat> %
;
; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -333,6 +377,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_non_inlinable(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_inlinable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -346,6 +392,8 @@ define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_non_inlinable(<16 x bfloa
;
; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_non_inlinable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b32_e32 v18, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -365,6 +413,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -374,6 +424,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[18:25], v[0:7], v[8:15], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -389,6 +441,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -403,6 +457,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -431,6 +487,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -446,6 +504,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -475,6 +535,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -484,6 +546,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[18:25], v[0:7], v[8:15], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -499,6 +563,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -513,6 +579,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -541,6 +609,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -556,6 +626,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -585,6 +657,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -594,6 +668,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[18:25], v[0:7], v[8:15], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -609,6 +685,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -623,6 +701,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -651,6 +731,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -666,6 +748,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -695,6 +779,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -704,6 +790,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[18:25], v[0:7], v[8:15], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -719,6 +807,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -733,6 +823,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -761,6 +853,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -776,6 +870,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -805,6 +901,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -812,6 +910,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[18:21], v[0:7], v[8:15], 1.0
; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -825,6 +925,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -836,6 +938,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -856,6 +960,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -867,6 +973,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -887,6 +995,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], 1.0 neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -894,6 +1004,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[18:21], v[0:7], v[8:15], 1.0 neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -907,6 +1019,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -918,6 +1032,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -938,6 +1054,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -949,6 +1067,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -969,6 +1089,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -976,6 +1098,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[18:21], v[0:7], v[8:15], 1.0
; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -989,6 +1113,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1000,6 +1126,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -1020,6 +1148,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1031,6 +1161,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -1051,6 +1183,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -1058,6 +1192,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8(<8 x i32> %A, <8 x i32> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[18:21], v[0:7], v[8:15], 1.0
; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -1071,6 +1207,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1082,6 +1220,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_non_splat(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -1102,6 +1242,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1113,6 +1255,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_non_inlineable(<8 x i32> %
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -1133,6 +1277,8 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x64_iu8(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], 1
; GFX1250-NEXT: s_clause 0x1
@@ -1142,6 +1288,8 @@ define amdgpu_ps void @test_wmma_i32_16x16x64_iu8(<8 x i32> %A, <8 x i32> %B, pt
;
; GISEL-LABEL: test_wmma_i32_16x16x64_iu8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[18:25], v[0:7], v[8:15], 1
; GISEL-NEXT: s_clause 0x1
@@ -1157,6 +1305,8 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_non_splat(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1 :: v_dual_mov_b32 v20, 2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -1171,6 +1321,8 @@ define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_non_splat(<8 x i32> %A, <8 x i
;
; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1
; GISEL-NEXT: s_mov_b32 s2, 2
@@ -1199,6 +1351,8 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_non_inlineable(<8 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x80
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -1214,6 +1368,8 @@ define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_non_inlineable(<8 x i32> %A, <
;
; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_movk_i32 s0, 0x80
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1243,6 +1399,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_f16(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -1252,6 +1410,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_f16(<16 x half> %A, <16 x half> %B
;
; GISEL-LABEL: test_wmma_f32_16x16x32_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[18:25], v[0:7], v[8:15], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -1267,6 +1427,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_f16_non_splat(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v18, 1.0 :: v_dual_mov_b32 v20, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -1281,6 +1443,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_f16_non_splat(<16 x half> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x32_f16_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -1309,6 +1473,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_f16_non_inlineable(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -1324,6 +1490,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_f16_non_inlineable(<16 x half> %A,
;
; GISEL-LABEL: test_wmma_f32_16x16x32_f16_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1353,6 +1521,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x32_f16(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x32_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], 1.0
; GFX1250-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -1360,6 +1530,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x32_f16(<16 x half> %A, <16 x half> %B
;
; GISEL-LABEL: test_wmma_f16_16x16x32_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[18:21], v[0:7], v[8:15], 1.0
; GISEL-NEXT: global_store_b128 v[16:17], v[18:21], off
@@ -1373,6 +1545,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x32_f16_non_splat(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1384,6 +1558,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x32_f16_non_splat(<16 x half> %A, <16
;
; GISEL-LABEL: test_wmma_f16_16x16x32_f16_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -1404,6 +1580,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x32_f16_non_inlineable(<16 x half> %A, <16 x half> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v18, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1415,6 +1593,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x32_f16_non_inlineable(<16 x half> %A,
;
; GISEL-LABEL: test_wmma_f16_16x16x32_f16_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -1435,6 +1615,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -1444,6 +1626,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -1459,6 +1643,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v35, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -1473,6 +1659,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_non_splat(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s1, 2.0
@@ -1501,6 +1689,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -1516,6 +1706,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1545,6 +1737,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -1554,6 +1748,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GISEL-NEXT: s_clause 0x1
@@ -1569,6 +1765,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v35, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -1583,6 +1781,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_non_splat(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s1, 2.0
@@ -1611,6 +1811,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x40400000
; GFX1250-NEXT: v_mov_b32_e32 v43, 0x64
@@ -1627,6 +1829,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_non_inlineable(<16 x
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: v_mov_b32_e32 v42, 0x64
@@ -1657,6 +1861,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4(<16 x i32> %A, <16 x i32> %B, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x1
@@ -1666,6 +1872,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4(<16 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[34:41], v[0:15], v[16:31], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GISEL-NEXT: s_clause 0x1
@@ -1681,6 +1889,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v35, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -1695,6 +1905,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_non_splat(<16 x i3
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s1, 2.0
@@ -1723,6 +1935,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x40400000
; GFX1250-NEXT: v_mov_b64_e32 v[42:43], 0x65
@@ -1740,6 +1954,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_non_inlineable(<16
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: v_mov_b64_e32 v[42:43], 0x64
@@ -1770,6 +1986,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -1777,6 +1995,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[34:37], v[0:15], v[16:31], 1.0
; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -1790,6 +2010,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1801,6 +2023,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -1821,6 +2045,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1832,6 +2058,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -1852,6 +2080,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], 1.0 neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -1859,6 +2089,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[34:37], v[0:15], v[16:31], 1.0 neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -1872,6 +2104,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1883,6 +2117,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -1903,6 +2139,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1914,6 +2152,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -1934,6 +2174,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -1941,6 +2183,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[34:37], v[0:15], v[16:31], 1.0
; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -1954,6 +2198,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1965,6 +2211,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -1985,6 +2233,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -1996,6 +2246,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -2016,6 +2268,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -2023,6 +2277,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[34:37], v[0:15], v[16:31], 1.0
; GISEL-NEXT: global_store_b128 v[32:33], v[34:37], off
@@ -2036,6 +2292,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x3c003c00
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -2047,6 +2305,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x3c003c00
; GISEL-NEXT: s_mov_b32 s1, 0x3c004000
@@ -2067,6 +2327,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x42004200
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -2078,6 +2340,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x42004200
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
@@ -2098,6 +2362,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -2107,6 +2373,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[34:41], v[0:15], v[16:31], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -2122,6 +2390,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -2136,6 +2406,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -2164,6 +2436,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -2179,6 +2453,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2208,6 +2484,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -2217,6 +2495,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[34:41], v[0:15], v[16:31], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -2232,6 +2512,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -2246,6 +2528,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -2274,6 +2558,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -2289,6 +2575,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2318,6 +2606,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -2327,6 +2617,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[34:41], v[0:15], v[16:31], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -2342,6 +2634,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -2356,6 +2650,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -2384,6 +2680,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -2399,6 +2697,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2428,6 +2728,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], 1.0
; GFX1250-NEXT: s_clause 0x1
@@ -2437,6 +2739,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8(<16 x i32> %A, <16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[34:41], v[0:15], v[16:31], 1.0
; GISEL-NEXT: s_clause 0x1
@@ -2452,6 +2756,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_non_splat(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v34, 1.0 :: v_dual_mov_b32 v36, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
@@ -2466,6 +2772,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -2494,6 +2802,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_non_inlineable(<16 x i32> %A, <16 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v34, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
@@ -2509,6 +2819,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_non_inlineable(<16 x i32>
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2538,6 +2850,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_32x16x128_f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0
; GFX1250-NEXT: s_clause 0x3
@@ -2549,6 +2863,8 @@ define amdgpu_ps void @test_wmma_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, p
;
; GISEL-LABEL: test_wmma_f32_32x16x128_f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0
; GISEL-NEXT: s_clause 0x3
@@ -2566,6 +2882,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_32x16x128_f4_non_splat(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v26, 1.0 :: v_dual_mov_b32 v28, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2587,6 +2905,8 @@ define amdgpu_ps void @test_wmma_f32_32x16x128_f4_non_splat(<16 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f32_32x16x128_f4_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s2, 2.0
@@ -2629,6 +2949,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_32x16x128_f4_non_inlineable(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v26, 0x40400000
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2651,6 +2973,8 @@ define amdgpu_ps void @test_wmma_f32_32x16x128_f4_non_inlineable(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_f32_32x16x128_f4_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2694,6 +3018,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, i32 inreg %scale_src0, i32 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x3
@@ -2705,6 +3031,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4(<16 x i32> %A, <8 x i32>
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s0, s1 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GISEL-NEXT: s_clause 0x3
@@ -2722,6 +3050,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_non_splat(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v26, 1.0 :: v_dual_mov_b32 v27, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2743,6 +3073,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_non_splat(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s1, 2.0
@@ -2785,6 +3117,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_non_inlineable(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v26, 0x40400000
; GFX1250-NEXT: v_mov_b32_e32 v43, 0x64
@@ -2808,6 +3142,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_non_inlineable(<16 x i32
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: v_mov_b32_e32 v42, 0x64
@@ -2852,6 +3188,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4(<16 x i32> %A, <8 x i32> %B, i64 inreg %scale_src0, i64 inreg %scale_src1, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GFX1250-NEXT: s_clause 0x3
@@ -2863,6 +3201,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4(<16 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[26:41], v[0:15], v[16:23], 1.0, s[0:1], s[2:3] matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 matrix_a_reuse
; GISEL-NEXT: s_clause 0x3
@@ -2880,6 +3220,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_non_splat(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_splat:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v26, 1.0 :: v_dual_mov_b32 v27, 2.0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2901,6 +3243,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_non_splat(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 1.0
; GISEL-NEXT: s_mov_b32 s1, 2.0
@@ -2943,6 +3287,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_non_inlineable(<16 x i32> %A, <8 x i32> %B, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_inlineable:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v26, 0x40400000
; GFX1250-NEXT: v_mov_b64_e32 v[42:43], 0x65
@@ -2967,6 +3313,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_non_inlineable(<16 x i
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s0, 0x40400000
; GISEL-NEXT: v_mov_b64_e32 v[42:43], 0x64
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll
index 44820e8841586..eebd275ac8513 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @test_wmma_f64_16x16x4_f64(<2 x double> %A, <2 x double> %B, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], 1.0
; GFX1251-NEXT: s_clause 0x3
@@ -16,6 +18,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64(<2 x double> %A, <2 x double> %
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[10:25], v[0:3], v[4:7], 1.0
; GISEL-NEXT: s_clause 0x3
@@ -33,6 +37,8 @@ bb:
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_non_splat(<2 x double> %A, <2 x double> %B, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_non_splat:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0x3ff00000
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -54,6 +60,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_non_splat(<2 x double> %A, <2 x
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_non_splat:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b64 s[0:1], 1.0
; GISEL-NEXT: s_mov_b64 s[4:5], 2.0
@@ -88,6 +96,8 @@ bb:
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_non_inlineable(<2 x double> %A, <2 x double> %B, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_non_inlineable:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0x40080000
; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -109,6 +119,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_non_inlineable(<2 x double> %A,
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_non_inlineable:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b64 s[0:1], 0x4008000000000000
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1250.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1250.w32.ll
index 96a329d72aec9..647408be94455 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1250.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1250.w32.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @test_wmma_f32_16x16x4_f32_negC(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -14,6 +16,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x4_f32_negC(<2 x float> %A, <2 x float
;
; GISEL-LABEL: test_wmma_f32_16x16x4_f32_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -29,6 +33,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x4_f32_neg_absC(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -38,6 +44,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x4_f32_neg_absC(<2 x float> %A, <2 x f
;
; GISEL-LABEL: test_wmma_f32_16x16x4_f32_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -53,6 +61,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x4_f32_ignoreC(<2 x float> %A, <2 x float> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x4_f32_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11]
; GFX1250-NEXT: s_clause 0x1
@@ -62,6 +72,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x4_f32_ignoreC(<2 x float> %A, <2 x fl
;
; GISEL-LABEL: test_wmma_f32_16x16x4_f32_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x4_f32 v[4:11], v[0:1], v[2:3], v[4:11]
; GISEL-NEXT: s_clause 0x1
@@ -77,6 +89,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -86,6 +100,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -101,6 +117,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -110,6 +128,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <1
;
; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -125,6 +145,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_bf16_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23]
; GFX1250-NEXT: s_clause 0x1
@@ -134,6 +156,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x32_bf16_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_bf16 v[16:23], v[0:7], v[8:15], v[16:23]
; GISEL-NEXT: s_clause 0x1
@@ -149,6 +173,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -156,6 +182,8 @@ define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x
;
; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -169,6 +197,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -176,6 +206,8 @@ define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <
;
; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -189,6 +221,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16_16x16x32_bf16_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -196,6 +230,8 @@ define amdgpu_ps void @test_wmma_bf16_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <1
;
; GISEL-LABEL: test_wmma_bf16_16x16x32_bf16_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16_16x16x32_bf16 v[16:19], v[0:7], v[8:15], v[16:19]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -209,6 +245,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_negC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -216,6 +254,8 @@ define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_negC(<16 x bfloat> %A, <1
;
; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -229,6 +269,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_neg_absC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -236,6 +278,8 @@ define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_neg_absC(<16 x bfloat> %A
;
; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -249,6 +293,8 @@ bb:
define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_ignoreC(<16 x bfloat> %A, <16 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_bf16f32_16x16x32_bf16_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23]
; GFX1250-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -256,6 +302,8 @@ define amdgpu_ps void @test_wmma_bf16f32_16x16x32_bf16_ignoreC(<16 x bfloat> %A,
;
; GISEL-LABEL: test_wmma_bf16f32_16x16x32_bf16_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_bf16f32_16x16x32_bf16 v[26:29], v[0:7], v[8:15], v[16:23]
; GISEL-NEXT: global_store_b128 v[24:25], v[26:29], off
@@ -269,6 +317,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -278,6 +328,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -293,6 +345,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -302,6 +356,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -317,6 +373,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]
; GFX1250-NEXT: s_clause 0x1
@@ -326,6 +384,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_fp8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]
; GISEL-NEXT: s_clause 0x1
@@ -341,6 +401,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -350,6 +412,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -365,6 +429,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -374,6 +440,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -389,6 +457,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_fp8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]
; GFX1250-NEXT: s_clause 0x1
@@ -398,6 +468,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_fp8_bf8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f32_16x16x64_fp8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_fp8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]
; GISEL-NEXT: s_clause 0x1
@@ -413,6 +485,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -422,6 +496,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -437,6 +513,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -446,6 +524,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -461,6 +541,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]
; GFX1250-NEXT: s_clause 0x1
@@ -470,6 +552,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_fp8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_fp8 v[16:23], v[0:7], v[8:15], v[16:23]
; GISEL-NEXT: s_clause 0x1
@@ -485,6 +569,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -494,6 +580,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -509,6 +597,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -518,6 +608,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -533,6 +625,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x64_bf8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]
; GFX1250-NEXT: s_clause 0x1
@@ -542,6 +636,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x64_bf8_bf8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f32_16x16x64_bf8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x64_bf8_bf8 v[16:23], v[0:7], v[8:15], v[16:23]
; GISEL-NEXT: s_clause 0x1
@@ -557,6 +653,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -564,6 +662,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -577,6 +677,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -584,6 +686,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -597,6 +701,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -604,6 +710,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_fp8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -617,6 +725,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -624,6 +734,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -637,6 +749,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -644,6 +758,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -657,6 +773,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_fp8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -664,6 +782,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_fp8_bf8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f16_16x16x64_fp8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_fp8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -677,6 +797,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -684,6 +806,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -697,6 +821,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -704,6 +830,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -717,6 +845,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -724,6 +854,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_fp8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_fp8 v[16:19], v[0:7], v[8:15], v[16:19]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -737,6 +869,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_negC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -744,6 +878,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_negC(<8 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -757,6 +893,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_neg_absC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -764,6 +902,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_neg_absC(<8 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -777,6 +917,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_ignoreC(<8 x i32> %A, <8 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x64_bf8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -784,6 +926,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x64_bf8_bf8_ignoreC(<8 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_f16_16x16x64_bf8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x64_bf8_bf8 v[16:19], v[0:7], v[8:15], v[16:19]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -797,6 +941,8 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_signedA(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_signedA:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0]
; GFX1250-NEXT: s_clause 0x1
@@ -806,6 +952,8 @@ define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_signedA(<8 x i32> %A, <8 x i32
;
; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_signedA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[1,0,0]
; GISEL-NEXT: s_clause 0x1
@@ -821,6 +969,8 @@ bb:
define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_signedB(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_i32_16x16x64_iu8_signedB:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0]
; GFX1250-NEXT: s_clause 0x1
@@ -830,6 +980,8 @@ define amdgpu_ps void @test_wmma_i32_16x16x64_iu8_signedB(<8 x i32> %A, <8 x i32
;
; GISEL-LABEL: test_wmma_i32_16x16x64_iu8_signedB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_i32_16x16x64_iu8 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,1,0]
; GISEL-NEXT: s_clause 0x1
@@ -845,6 +997,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_f16_negC(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -854,6 +1008,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_f16_negC(<16 x half> %A, <16 x hal
;
; GISEL-LABEL: test_wmma_f32_16x16x32_f16_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -869,6 +1025,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_f16_neg_absC(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -878,6 +1036,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_f16_neg_absC(<16 x half> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x32_f16_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -893,6 +1053,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x32_f16_ignoreC(<16 x half> %A, <16 x half> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x32_f16_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23]
; GFX1250-NEXT: s_clause 0x1
@@ -902,6 +1064,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x32_f16_ignoreC(<16 x half> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x32_f16_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[0:7], v[8:15], v[16:23]
; GISEL-NEXT: s_clause 0x1
@@ -917,6 +1081,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x32_f16_negC(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -924,6 +1090,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x32_f16_negC(<16 x half> %A, <16 x hal
;
; GISEL-LABEL: test_wmma_f16_16x16x32_f16_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -937,6 +1105,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x32_f16_neg_absC(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -944,6 +1114,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x32_f16_neg_absC(<16 x half> %A, <16 x
;
; GISEL-LABEL: test_wmma_f16_16x16x32_f16_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -957,6 +1129,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x32_f16_ignoreC(<16 x half> %A, <16 x half> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x32_f16_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19]
; GFX1250-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -964,6 +1138,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x32_f16_ignoreC(<16 x half> %A, <16 x
;
; GISEL-LABEL: test_wmma_f16_16x16x32_f16_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x32_f16 v[16:19], v[0:7], v[8:15], v[16:19]
; GISEL-NEXT: global_store_b128 v[20:21], v[16:19], off
@@ -977,6 +1153,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -986,6 +1164,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1001,6 +1181,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1010,6 +1192,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_neg_absC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1025,6 +1209,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_f8f6f4_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39]
; GFX1250-NEXT: s_clause 0x1
@@ -1034,6 +1220,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %A, <16
;
; GISEL-LABEL: test_wmma_f32_16x16x128_f8f6f4_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39]
; GISEL-NEXT: s_clause 0x1
@@ -1049,6 +1237,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1058,6 +1248,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_negC(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1073,6 +1265,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1082,6 +1276,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_neg_absC(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1097,6 +1293,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x1
@@ -1106,6 +1304,8 @@ define amdgpu_ps void @test_wmma_scale_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %
;
; GISEL-LABEL: test_wmma_scale_f32_16x16x128_f8f6f4_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x1
@@ -1121,6 +1321,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1130,6 +1332,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_negC(<16 x i32> %A
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1145,6 +1349,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1154,6 +1360,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_neg_absC(<16 x i32
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1169,6 +1377,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x1
@@ -1178,6 +1388,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_16x16x128_f8f6f4_ignoreC(<16 x i32>
;
; GISEL-LABEL: test_wmma_scale16_f32_16x16x128_f8f6f4_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_16x16x128_f8f6f4 v[32:39], v[0:15], v[16:31], v[32:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x1
@@ -1193,6 +1405,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1200,6 +1414,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1213,6 +1429,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1220,6 +1438,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1233,6 +1453,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1240,6 +1462,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_fp8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1253,6 +1477,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1260,6 +1486,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1273,6 +1501,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1280,6 +1510,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1293,6 +1525,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_fp8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1300,6 +1534,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_fp8_bf8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f16_16x16x128_fp8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_fp8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1313,6 +1549,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1320,6 +1558,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1333,6 +1573,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1340,6 +1582,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1353,6 +1597,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1360,6 +1606,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_fp8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_fp8 v[32:35], v[0:15], v[16:31], v[32:35]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1373,6 +1621,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1380,6 +1630,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1393,6 +1645,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1400,6 +1654,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1413,6 +1669,8 @@ bb:
define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f16_16x16x128_bf8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]
; GFX1250-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1420,6 +1678,8 @@ define amdgpu_ps void @test_wmma_f16_16x16x128_bf8_bf8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f16_16x16x128_bf8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f16_16x16x128_bf8_bf8 v[32:35], v[0:15], v[16:31], v[32:35]
; GISEL-NEXT: global_store_b128 v[36:37], v[32:35], off
@@ -1433,6 +1693,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1442,6 +1704,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1457,6 +1721,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1466,6 +1732,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1481,6 +1749,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]
; GFX1250-NEXT: s_clause 0x1
@@ -1490,6 +1760,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_fp8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]
; GISEL-NEXT: s_clause 0x1
@@ -1505,6 +1777,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1514,6 +1788,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1529,6 +1805,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1538,6 +1816,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1553,6 +1833,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_fp8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]
; GFX1250-NEXT: s_clause 0x1
@@ -1562,6 +1844,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_fp8_bf8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f32_16x16x128_fp8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_fp8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]
; GISEL-NEXT: s_clause 0x1
@@ -1577,6 +1861,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1586,6 +1872,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1601,6 +1889,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1610,6 +1900,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1625,6 +1917,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_fp8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]
; GFX1250-NEXT: s_clause 0x1
@@ -1634,6 +1928,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_fp8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_fp8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_fp8 v[32:39], v[0:15], v[16:31], v[32:39]
; GISEL-NEXT: s_clause 0x1
@@ -1649,6 +1945,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_negC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1658,6 +1956,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_negC(<16 x i32> %A, <16 x
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1673,6 +1973,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_neg_absC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x1
@@ -1682,6 +1984,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_neg_absC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x1
@@ -1697,6 +2001,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_ignoreC(<16 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_16x16x128_bf8_bf8_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]
; GFX1250-NEXT: s_clause 0x1
@@ -1706,6 +2012,8 @@ define amdgpu_ps void @test_wmma_f32_16x16x128_bf8_bf8_ignoreC(<16 x i32> %A, <1
;
; GISEL-LABEL: test_wmma_f32_16x16x128_bf8_bf8_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_16x16x128_bf8_bf8 v[32:39], v[0:15], v[16:31], v[32:39]
; GISEL-NEXT: s_clause 0x1
@@ -1721,6 +2029,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x3
@@ -1732,6 +2042,8 @@ define amdgpu_ps void @test_wmma_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x i32>
;
; GISEL-LABEL: test_wmma_f32_32x16x128_f4_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -1749,6 +2061,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_32x16x128_f4_neg_absC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x3
@@ -1760,6 +2074,8 @@ define amdgpu_ps void @test_wmma_f32_32x16x128_f4_neg_absC(<16 x i32> %A, <8 x i
;
; GISEL-LABEL: test_wmma_f32_32x16x128_f4_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -1777,6 +2093,8 @@ bb:
define amdgpu_ps void @test_wmma_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_f32_32x16x128_f4_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]
; GFX1250-NEXT: s_clause 0x3
@@ -1788,6 +2106,8 @@ define amdgpu_ps void @test_wmma_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <8 x i3
;
; GISEL-LABEL: test_wmma_f32_32x16x128_f4_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39]
; GISEL-NEXT: s_clause 0x3
@@ -1805,6 +2125,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x3
@@ -1816,6 +2138,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -1833,6 +2157,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_neg_absC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x3
@@ -1844,6 +2170,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_neg_absC(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -1861,6 +2189,8 @@ bb:
define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale_f32_32x16x128_f4_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x3
@@ -1872,6 +2202,8 @@ define amdgpu_ps void @test_wmma_scale_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <
;
; GISEL-LABEL: test_wmma_scale_f32_32x16x128_f4_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x3
@@ -1889,6 +2221,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_negC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_negC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GFX1250-NEXT: s_clause 0x3
@@ -1900,6 +2234,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_negC(<16 x i32> %A, <8
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -1917,6 +2253,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_neg_absC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_neg_absC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1250-NEXT: s_clause 0x3
@@ -1928,6 +2266,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_neg_absC(<16 x i32> %A
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1 neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -1945,6 +2285,8 @@ bb:
define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_ignoreC(<16 x i32> %A, <8 x i32> %B, <16 x float> %C, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_wmma_scale16_f32_32x16x128_f4_ignoreC:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GFX1250-NEXT: s_clause 0x3
@@ -1956,6 +2298,8 @@ define amdgpu_ps void @test_wmma_scale16_f32_32x16x128_f4_ignoreC(<16 x i32> %A,
;
; GISEL-LABEL: test_wmma_scale16_f32_32x16x128_f4_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_scale16_f32_32x16x128_f4 v[24:39], v[0:15], v[16:23], v[24:39], 2, 4 matrix_a_scale:MATRIX_SCALE_ROW1 matrix_b_scale:MATRIX_SCALE_ROW1
; GISEL-NEXT: s_clause 0x3
@@ -1973,6 +2317,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16_negA(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16_negA:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -1983,6 +2329,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16_negA(<16 x bfloat> %A, <32
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16_negA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -1999,6 +2347,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16_negB(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16_negB:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2009,6 +2359,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16_negB(<16 x bfloat> %A, <32
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16_negB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2025,6 +2377,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16_negA(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16_negA:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2033,6 +2387,8 @@ define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16_negA(<16 x bfloat> %A, <32
;
; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16_negA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GISEL-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2047,6 +2403,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16_negB(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16_negB:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2055,6 +2413,8 @@ define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16_negB(<16 x bfloat> %A, <32
;
; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16_negB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GISEL-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2069,6 +2429,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16_negA(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negA:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2079,6 +2441,8 @@ define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16_negA(<16 x bfloat> %A,
;
; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2095,6 +2459,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16_negB(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negB:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2105,6 +2471,8 @@ define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16_negB(<16 x bfloat> %A,
;
; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16_negB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GISEL-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2121,6 +2489,8 @@ bb:
define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_signedA(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, i64 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_signedA:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[1,0,0]
; GFX1250-NEXT: s_clause 0x1
@@ -2130,6 +2500,8 @@ define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_signedA(<8 x i32> %A, <16 x
;
; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_signedA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[1,0,0]
; GISEL-NEXT: s_clause 0x1
@@ -2145,6 +2517,8 @@ bb:
define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_signedB(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, i64 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_signedB:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[0,1,0]
; GFX1250-NEXT: s_clause 0x1
@@ -2154,6 +2528,8 @@ define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_signedB(<8 x i32> %A, <16 x
;
; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_signedB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] neg_lo:[0,1,0]
; GISEL-NEXT: s_clause 0x1
@@ -2169,6 +2545,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x64_f16_negA(<16 x half> %A, <32 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16_negA:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2179,6 +2557,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_f16_negA(<16 x half> %A, <32 x h
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_f16_negA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GISEL-NEXT: v_dual_mov_b32 v36, v33 :: v_dual_mov_b32 v37, v34
@@ -2195,6 +2575,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x64_f16_negB(<16 x half> %A, <32 x half> %B, <8 x float> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16_negB:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GFX1250-NEXT: v_dual_mov_b32 v35, v34 :: v_dual_mov_b32 v34, v33
@@ -2205,6 +2587,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_f16_negB(<16 x half> %A, <32 x h
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_f16_negB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GISEL-NEXT: v_dual_mov_b32 v36, v33 :: v_dual_mov_b32 v37, v34
@@ -2221,6 +2605,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x64_f16_negA(<16 x half> %A, <32 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16_negA:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2229,6 +2615,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x64_f16_negA(<16 x half> %A, <32 x h
;
; GISEL-LABEL: test_swmmac_f16_16x16x64_f16_negA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[1,0,0] neg_hi:[1,0,0]
; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v30
@@ -2243,6 +2631,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x64_f16_negB(<16 x half> %A, <32 x half> %B, <8 x half> %C, i16 %Index, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16_negB:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GFX1250-NEXT: v_dual_mov_b32 v31, v30 :: v_dual_mov_b32 v30, v29
@@ -2251,6 +2641,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x64_f16_negB(<16 x half> %A, <32 x h
;
; GISEL-LABEL: test_swmmac_f16_16x16x64_f16_negB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 neg_lo:[0,1,0] neg_hi:[0,1,0]
; GISEL-NEXT: v_dual_mov_b32 v32, v29 :: v_dual_mov_b32 v33, v30
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll
index fcc3c98e78534..d17e895d05f45 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negA(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_negA:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[1,0,0]
; GFX1251-NEXT: s_clause 0x3
@@ -16,6 +18,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negA(<2 x double> %A, <2 x doub
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_negA:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[1,0,0]
; GISEL-NEXT: s_clause 0x3
@@ -33,6 +37,8 @@ bb:
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negB(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_negB:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,1,0]
; GFX1251-NEXT: s_clause 0x3
@@ -44,6 +50,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negB(<2 x double> %A, <2 x doub
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_negB:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,1,0]
; GISEL-NEXT: s_clause 0x3
@@ -61,6 +69,8 @@ bb:
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negC(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_negC:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1]
; GFX1251-NEXT: s_clause 0x3
@@ -72,6 +82,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_negC(<2 x double> %A, <2 x doub
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_negC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -89,6 +101,8 @@ bb:
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_neg_absC(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_neg_absC:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GFX1251-NEXT: s_clause 0x3
@@ -100,6 +114,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_neg_absC(<2 x double> %A, <2 x
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_neg_absC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23] neg_lo:[0,0,1] neg_hi:[0,0,1]
; GISEL-NEXT: s_clause 0x3
@@ -117,6 +133,8 @@ bb:
define amdgpu_ps void @test_wmma_f64_16x16x4_f64_ignoreC(<2 x double> %A, <2 x double> %B, <8 x double> %C, ptr addrspace(1) %out) {
; GFX1251-LABEL: test_wmma_f64_16x16x4_f64_ignoreC:
; GFX1251: ; %bb.0: ; %bb
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23]
; GFX1251-NEXT: s_clause 0x3
@@ -128,6 +146,8 @@ define amdgpu_ps void @test_wmma_f64_16x16x4_f64_ignoreC(<2 x double> %A, <2 x d
;
; GISEL-LABEL: test_wmma_f64_16x16x4_f64_ignoreC:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_wmma_f64_16x16x4_f64 v[8:23], v[0:3], v[4:7], v[8:23]
; GISEL-NEXT: s_clause 0x3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.index.gfx1250.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.index.gfx1250.w32.ll
index fafc4526065d6..f5a4338c0c732 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.index.gfx1250.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.index.gfx1250.w32.ll
@@ -5,6 +5,8 @@
define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v32, v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -16,6 +18,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfl
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b32 v32, v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -36,6 +40,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v28, v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -45,6 +51,8 @@ define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16(<16 x bfloat> %A, <32 x bf
;
; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b32 v28, v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -63,6 +71,8 @@ bb:
define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v32, v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -74,6 +84,8 @@ define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16(<16 x bfloat> %A, <32 x
;
; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b32 v32, v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -94,6 +106,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -106,6 +120,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -128,6 +144,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -140,6 +158,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -161,6 +181,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -172,6 +194,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -190,6 +214,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -202,6 +228,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -224,6 +252,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -236,6 +266,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -257,6 +289,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -268,6 +302,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -286,6 +322,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -298,6 +336,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -320,6 +360,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -332,6 +374,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -353,6 +397,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -364,6 +410,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -382,6 +430,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -394,6 +444,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -416,6 +468,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -428,6 +482,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -449,6 +505,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -460,6 +518,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -478,6 +538,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -488,6 +550,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -508,6 +572,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -518,6 +584,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -537,6 +605,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -546,6 +616,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -562,6 +634,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -572,6 +646,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -592,6 +668,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -602,6 +680,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -621,6 +701,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -630,6 +712,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -646,6 +730,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -656,6 +742,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -676,6 +764,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -686,6 +776,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -705,6 +797,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -714,6 +808,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -730,6 +826,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -740,6 +838,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_i32_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -760,6 +860,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -770,6 +872,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_i64_index(<8 x i32> %A,
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -789,6 +893,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -798,6 +904,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_v2i32_index(<8 x i32> %
;
; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -814,6 +922,8 @@ bb:
define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -826,6 +936,8 @@ define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_i32_index(<8 x i32> %A, <16
;
; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -848,6 +960,8 @@ bb:
define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_i64_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -860,6 +974,8 @@ define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_i64_index(<8 x i32> %A, <16
;
; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_i64_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -881,6 +997,8 @@ bb:
define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_v2i32_index(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_v2i32_index:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -892,6 +1010,8 @@ define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_v2i32_index(<8 x i32> %A, <
;
; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_v2i32_index:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -910,6 +1030,8 @@ bb:
define amdgpu_ps void @test_swmmac_f32_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v32, v[32:33], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -921,6 +1043,8 @@ define amdgpu_ps void @test_swmmac_f32_16x16x64_f16(<16 x half> %A, <32 x half>
;
; GISEL-LABEL: test_swmmac_f32_16x16x64_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b32 v32, v[32:33], off
; GISEL-NEXT: s_wait_loadcnt 0x0
@@ -941,6 +1065,8 @@ bb:
define amdgpu_ps void @test_swmmac_f16_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v28, v[28:29], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -950,6 +1076,8 @@ define amdgpu_ps void @test_swmmac_f16_16x16x64_f16(<16 x half> %A, <32 x half>
;
; GISEL-LABEL: test_swmmac_f16_16x16x64_f16:
; GISEL: ; %bb.0: ; %bb
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: global_load_b32 v28, v[28:29], off
; GISEL-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
index 7ded324418fc1..b6d4957bba82b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.cos.bf16(bfloat) #0
define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-LABEL: cos_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; FAKE16-NEXT: s_mov_b32 s3, 0x3e230000
@@ -25,6 +27,8 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; REAL16-LABEL: cos_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; REAL16-NEXT: s_mov_b32 s3, 0x3e230000
@@ -44,6 +48,8 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @cos_bf16_constant_4(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: cos_bf16_constant_4:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_cos_bf16_e32 v0, 0x3f23
@@ -54,6 +60,8 @@ define amdgpu_kernel void @cos_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: cos_bf16_constant_4:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_cos_bf16_e32 v0.l, 0x3f23
@@ -69,6 +77,8 @@ define amdgpu_kernel void @cos_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @cos_bf16_constant_100(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: cos_bf16_constant_100:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_cos_bf16_e32 v0, 0x417f
@@ -79,6 +89,8 @@ define amdgpu_kernel void @cos_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: cos_bf16_constant_100:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_cos_bf16_e32 v0.l, 0x417f
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll
index f042474df4fa7..e0768cd6e9d19 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll
@@ -17,6 +17,8 @@
define amdgpu_ps void @prefetch_data_sgpr(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -24,6 +26,8 @@ define amdgpu_ps void @prefetch_data_sgpr(ptr addrspace(4) inreg %ptr) {
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_data s[0:1], 0x0, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -44,6 +48,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] offset:512 scope:SCOPE_SYS
@@ -51,6 +57,8 @@ define amdgpu_ps void @prefetch_data_sgpr_offset(ptr addrspace(4) inreg %ptr) {
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_offset:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_data s[0:1], 0x200, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -74,6 +82,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_max_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_max_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] offset:8388607 scope:SCOPE_SYS
@@ -81,6 +91,8 @@ define amdgpu_ps void @prefetch_data_sgpr_max_offset(ptr addrspace(4) inreg %ptr
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_max_offset:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_data s[0:1], 0x7fffff, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -102,6 +114,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_min_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_min_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] offset:-8388608 scope:SCOPE_SYS
@@ -109,6 +123,8 @@ define amdgpu_ps void @prefetch_data_sgpr_min_offset(ptr addrspace(4) inreg %ptr
;
; GFX1250-SPREFETCH-SDAG-LABEL: prefetch_data_sgpr_min_offset:
; GFX1250-SPREFETCH-SDAG: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-SDAG-NEXT: global_wb
+; GFX1250-SPREFETCH-SDAG-NEXT: v_nop
; GFX1250-SPREFETCH-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-SDAG-NEXT: s_mov_b64 s[2:3], 0xffffffffff800000
; GFX1250-SPREFETCH-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -131,6 +147,8 @@ define amdgpu_ps void @prefetch_data_sgpr_min_offset(ptr addrspace(4) inreg %ptr
;
; GFX1250-SPREFETCH-GISEL-LABEL: prefetch_data_sgpr_min_offset:
; GFX1250-SPREFETCH-GISEL: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-GISEL-NEXT: global_wb
+; GFX1250-SPREFETCH-GISEL-NEXT: v_nop
; GFX1250-SPREFETCH-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_u32 s0, s0, 0xff800000
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_ci_u32 s1, s1, -1
@@ -152,6 +170,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_too_large_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_too_large_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -159,6 +179,8 @@ define amdgpu_ps void @prefetch_data_sgpr_too_large_offset(ptr addrspace(4) inre
;
; GFX1250-SPREFETCH-SDAG-LABEL: prefetch_data_sgpr_too_large_offset:
; GFX1250-SPREFETCH-SDAG: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-SDAG-NEXT: global_wb
+; GFX1250-SPREFETCH-SDAG-NEXT: v_nop
; GFX1250-SPREFETCH-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x800000
; GFX1250-SPREFETCH-SDAG-NEXT: s_prefetch_data s[0:1], 0x0, null, 0
@@ -176,6 +198,8 @@ define amdgpu_ps void @prefetch_data_sgpr_too_large_offset(ptr addrspace(4) inre
;
; GFX1250-SPREFETCH-GISEL-LABEL: prefetch_data_sgpr_too_large_offset:
; GFX1250-SPREFETCH-GISEL: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-GISEL-NEXT: global_wb
+; GFX1250-SPREFETCH-GISEL-NEXT: v_nop
; GFX1250-SPREFETCH-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_u32 s0, s0, 0x800000
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
@@ -199,12 +223,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_global(ptr addrspace(1) %ptr) {
; GFX1250-LABEL: prefetch_data_vgpr_global:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_prefetch_b8 v[0:1], off scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_global:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: global_prefetch_b8 v[0:1], off scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -224,12 +252,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_flat(ptr %ptr) {
; GFX1250-LABEL: prefetch_data_vgpr_flat:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_flat:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -249,12 +281,16 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_vgpr_offset_global(ptr addrspace(1) inreg %ptr, i32 %offset) {
; GFX1250-LABEL: prefetch_data_sgpr_vgpr_offset_global:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_vgpr_offset_global:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -281,12 +317,16 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_vgpr_offset_flat(ptr inreg %ptr, i32 %offset) {
; GFX1250-LABEL: prefetch_data_sgpr_vgpr_offset_flat:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_prefetch_b8 v0, s[0:1] offset:128 scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_vgpr_offset_flat:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: flat_prefetch_b8 v0, s[0:1] offset:128 scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -316,11 +356,15 @@ entry:
define amdgpu_ps void @prefetch_data_lds(ptr addrspace(3) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_lds:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_lds:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_endpgm
;
@@ -339,11 +383,15 @@ entry:
define amdgpu_ps void @prefetch_data_scratch(ptr addrspace(5) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_scratch:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_scratch:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_endpgm
;
@@ -364,6 +412,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_flat(ptr inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_flat:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: flat_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -371,6 +421,8 @@ define amdgpu_ps void @prefetch_data_sgpr_flat(ptr inreg %ptr) {
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_flat:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_data s[0:1], 0x0, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -391,6 +443,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_global(ptr addrspace(1) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_global:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -398,6 +452,8 @@ define amdgpu_ps void @prefetch_data_sgpr_global(ptr addrspace(1) inreg %ptr) {
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_global:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_data s[0:1], 0x0, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -418,11 +474,15 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_constant_32bit(ptr addrspace(6) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_constant_32bit:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_constant_32bit:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_mov_b32 s1, 0
; GFX1250-SPREFETCH-NEXT: s_prefetch_data s[0:1], 0x0, null, 0
@@ -447,11 +507,15 @@ entry:
define amdgpu_ps void @prefetch_inst_sgpr(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_inst_sgpr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_inst_sgpr:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_inst s[0:1], 0x0, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -472,11 +536,15 @@ entry:
define amdgpu_ps void @prefetch_inst_sgpr_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_inst_sgpr_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_inst_sgpr_offset:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_inst s[0:1], 0x80, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -500,11 +568,15 @@ entry:
define amdgpu_ps void @prefetch_inst_sgpr_max_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_inst_sgpr_max_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_inst_sgpr_max_offset:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_prefetch_inst s[0:1], 0x7fffff, null, 0
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -526,11 +598,15 @@ entry:
define amdgpu_ps void @prefetch_inst_sgpr_min_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_inst_sgpr_min_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-SDAG-LABEL: prefetch_inst_sgpr_min_offset:
; GFX1250-SPREFETCH-SDAG: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-SDAG-NEXT: global_wb
+; GFX1250-SPREFETCH-SDAG-NEXT: v_nop
; GFX1250-SPREFETCH-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-SDAG-NEXT: s_mov_b64 s[2:3], 0xffffffffff800000
; GFX1250-SPREFETCH-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -553,6 +629,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_min_offset(ptr addrspace(4) inreg %ptr
;
; GFX1250-SPREFETCH-GISEL-LABEL: prefetch_inst_sgpr_min_offset:
; GFX1250-SPREFETCH-GISEL: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-GISEL-NEXT: global_wb
+; GFX1250-SPREFETCH-GISEL-NEXT: v_nop
; GFX1250-SPREFETCH-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_u32 s0, s0, 0xff800000
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_ci_u32 s1, s1, -1
@@ -574,11 +652,15 @@ entry:
define amdgpu_ps void @prefetch_inst_sgpr_too_large_offset(ptr addrspace(4) inreg %ptr) {
; GFX1250-LABEL: prefetch_inst_sgpr_too_large_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-SDAG-LABEL: prefetch_inst_sgpr_too_large_offset:
; GFX1250-SPREFETCH-SDAG: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-SDAG-NEXT: global_wb
+; GFX1250-SPREFETCH-SDAG-NEXT: v_nop
; GFX1250-SPREFETCH-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x800000
; GFX1250-SPREFETCH-SDAG-NEXT: s_prefetch_inst s[0:1], 0x0, null, 0
@@ -596,6 +678,8 @@ define amdgpu_ps void @prefetch_inst_sgpr_too_large_offset(ptr addrspace(4) inre
;
; GFX1250-SPREFETCH-GISEL-LABEL: prefetch_inst_sgpr_too_large_offset:
; GFX1250-SPREFETCH-GISEL: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-GISEL-NEXT: global_wb
+; GFX1250-SPREFETCH-GISEL-NEXT: v_nop
; GFX1250-SPREFETCH-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_u32 s0, s0, 0x800000
; GFX1250-SPREFETCH-GISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
@@ -619,12 +703,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_flat_dev(ptr %ptr) {
; GFX1250-LABEL: prefetch_data_vgpr_flat_dev:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_flat_dev:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_DEV
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -644,12 +732,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_flat_se(ptr %ptr) {
; GFX1250-LABEL: prefetch_data_vgpr_flat_se:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_SE
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_flat_se:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_SE
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -669,18 +761,24 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_flat_cu(ptr %ptr) {
; GL2-ONLY-LABEL: prefetch_data_vgpr_flat_cu:
; GL2-ONLY: ; %bb.0: ; %entry
+; GL2-ONLY-NEXT: global_wb
+; GL2-ONLY-NEXT: v_nop
; GL2-ONLY-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GL2-ONLY-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_SE
; GL2-ONLY-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_flat_cu:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: flat_prefetch_b8 v[0:1] scope:SCOPE_SE
; GFX1250-SPREFETCH-NEXT: s_endpgm
;
; SAFE-CU-LABEL: prefetch_data_vgpr_flat_cu:
; SAFE-CU: ; %bb.0: ; %entry
+; SAFE-CU-NEXT: global_wb
+; SAFE-CU-NEXT: v_nop
; SAFE-CU-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SAFE-CU-NEXT: flat_prefetch_b8 v[0:1]
; SAFE-CU-NEXT: s_endpgm
@@ -702,12 +800,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_flat_offset(ptr %ptr) {
; GFX1250-LABEL: prefetch_data_vgpr_flat_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_prefetch_b8 v[0:1] offset:512 scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_flat_offset:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: flat_prefetch_b8 v[0:1] offset:512 scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -728,12 +830,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_global_offset(ptr addrspace(1) %ptr) {
; GFX1250-LABEL: prefetch_data_vgpr_global_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_prefetch_b8 v[0:1], off offset:512 scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_global_offset:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: global_prefetch_b8 v[0:1], off offset:512 scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -754,12 +860,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_global_saddr(ptr addrspace(1) inreg %ptr, i32 %voffset) {
; GFX1250-LABEL: prefetch_data_vgpr_global_saddr:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_global_saddr:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -780,12 +890,16 @@ entry:
define amdgpu_ps void @prefetch_data_vgpr_global_saddr_offset(ptr addrspace(1) inreg %ptr, i32 %voffset) {
; GFX1250-LABEL: prefetch_data_vgpr_global_saddr_offset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] offset:128 scope:SCOPE_SYS
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_vgpr_global_saddr_offset:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: global_prefetch_b8 v0, s[0:1] offset:128 scope:SCOPE_SYS
; GFX1250-SPREFETCH-NEXT: s_endpgm
@@ -809,11 +923,15 @@ entry:
define amdgpu_ps void @prefetch_inst_vgpr_global(ptr addrspace(1) %ptr) {
; GFX1250-LABEL: prefetch_inst_vgpr_global:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_inst_vgpr_global:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_endpgm
;
@@ -832,11 +950,15 @@ entry:
define amdgpu_ps void @prefetch_inst_vgpr_flat(ptr %ptr) {
; GFX1250-LABEL: prefetch_inst_vgpr_flat:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
;
; GFX1250-SPREFETCH-LABEL: prefetch_inst_vgpr_flat:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: s_endpgm
;
@@ -857,6 +979,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_flat_force_vector(ptr inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_flat_force_vector:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: flat_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -864,6 +988,8 @@ define amdgpu_ps void @prefetch_data_sgpr_flat_force_vector(ptr inreg %ptr) {
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_flat_force_vector:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SPREFETCH-NEXT: flat_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -885,6 +1011,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_global_force_vector(ptr addrspace(1) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_global_force_vector:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -892,6 +1020,8 @@ define amdgpu_ps void @prefetch_data_sgpr_global_force_vector(ptr addrspace(1) i
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_global_force_vector:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SPREFETCH-NEXT: global_prefetch_b8 v0, s[0:1] scope:SCOPE_SYS
@@ -913,6 +1043,8 @@ entry:
define amdgpu_ps void @prefetch_data_sgpr_global_saddr_force_vector(ptr addrspace(1) inreg %ptr) {
; GFX1250-LABEL: prefetch_data_sgpr_global_saddr_force_vector:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: global_prefetch_b8 v0, s[0:1] offset:1024 scope:SCOPE_SYS
@@ -920,6 +1052,8 @@ define amdgpu_ps void @prefetch_data_sgpr_global_saddr_force_vector(ptr addrspac
;
; GFX1250-SPREFETCH-LABEL: prefetch_data_sgpr_global_saddr_force_vector:
; GFX1250-SPREFETCH: ; %bb.0: ; %entry
+; GFX1250-SPREFETCH-NEXT: global_wb
+; GFX1250-SPREFETCH-NEXT: v_nop
; GFX1250-SPREFETCH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SPREFETCH-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SPREFETCH-NEXT: global_prefetch_b8 v0, s[0:1] offset:1024 scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
index cc8c01a7afcd3..f85d3c94ffd32 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
@@ -11,6 +11,8 @@ declare bfloat @llvm.sin.bf16(bfloat) #0
define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-LABEL: sin_bf16:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; FAKE16-NEXT: s_mov_b32 s3, 0x3e230000
@@ -25,6 +27,8 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
;
; REAL16-LABEL: sin_bf16:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; REAL16-NEXT: s_mov_b32 s3, 0x3e230000
@@ -44,6 +48,8 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
define amdgpu_kernel void @sin_bf16_constant_4(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: sin_bf16_constant_4:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_sin_bf16_e32 v0, 0x3f23
@@ -54,6 +60,8 @@ define amdgpu_kernel void @sin_bf16_constant_4(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: sin_bf16_constant_4:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_sin_bf16_e32 v0.l, 0x3f23
@@ -69,6 +77,8 @@ define amdgpu_kernel void @sin_bf16_constant_4(ptr addrspace(1) %out) #1 {
define amdgpu_kernel void @sin_bf16_constant_100(ptr addrspace(1) %out) #1 {
; FAKE16-LABEL: sin_bf16_constant_100:
; FAKE16: ; %bb.0:
+; FAKE16-NEXT: global_wb
+; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; FAKE16-NEXT: v_sin_bf16_e32 v0, 0x417f
@@ -79,6 +89,8 @@ define amdgpu_kernel void @sin_bf16_constant_100(ptr addrspace(1) %out) #1 {
;
; REAL16-LABEL: sin_bf16_constant_100:
; REAL16: ; %bb.0:
+; REAL16-NEXT: global_wb
+; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_sin_bf16_e32 v0.l, 0x417f
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
index 1ad27d663f149..71fb1202f6d89 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
@@ -8,6 +8,8 @@ declare <2 x bfloat> @llvm.sqrt.v2bf16(<2 x bfloat> %a)
define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-LABEL: sqrt_bf16:
; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: global_wb
+; GFX12-TRUE16-NEXT: v_nop
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -27,6 +29,8 @@ define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
;
; GFX12-FAKE16-LABEL: sqrt_bf16:
; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: global_wb
+; GFX12-FAKE16-NEXT: v_nop
; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -53,6 +57,8 @@ entry:
define amdgpu_kernel void @sqrt_v2bf16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-LABEL: sqrt_v2bf16:
; GFX12-TRUE16: ; %bb.0: ; %entry
+; GFX12-TRUE16-NEXT: global_wb
+; GFX12-TRUE16-NEXT: v_nop
; GFX12-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -75,6 +81,8 @@ define amdgpu_kernel void @sqrt_v2bf16(ptr addrspace(1) %r, ptr addrspace(1) %a)
;
; GFX12-FAKE16-LABEL: sqrt_v2bf16:
; GFX12-FAKE16: ; %bb.0: ; %entry
+; GFX12-FAKE16-NEXT: global_wb
+; GFX12-FAKE16-NEXT: v_nop
; GFX12-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index d4363157e1238..916e86b9da7fd 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -79,6 +79,8 @@ define amdgpu_kernel void @constant_load_i1(ptr addrspace(1) %out, ptr addrspace
;
; GFX1250-LABEL: constant_load_i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -163,6 +165,8 @@ define amdgpu_kernel void @constant_load_v2i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-FAKE16-LABEL: constant_load_v2i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -174,6 +178,8 @@ define amdgpu_kernel void @constant_load_v2i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-REAL16-LABEL: constant_load_v2i1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -255,6 +261,8 @@ define amdgpu_kernel void @constant_load_v3i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-FAKE16-LABEL: constant_load_v3i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -266,6 +274,8 @@ define amdgpu_kernel void @constant_load_v3i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-REAL16-LABEL: constant_load_v3i1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -348,6 +358,8 @@ define amdgpu_kernel void @constant_load_v4i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-FAKE16-LABEL: constant_load_v4i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -359,6 +371,8 @@ define amdgpu_kernel void @constant_load_v4i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-REAL16-LABEL: constant_load_v4i1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -441,6 +455,8 @@ define amdgpu_kernel void @constant_load_v8i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-FAKE16-LABEL: constant_load_v8i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -452,6 +468,8 @@ define amdgpu_kernel void @constant_load_v8i1(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-REAL16-LABEL: constant_load_v8i1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -534,6 +552,8 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-FAKE16-LABEL: constant_load_v16i1:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, 0
@@ -545,6 +565,8 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-REAL16-LABEL: constant_load_v16i1:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -611,6 +633,8 @@ define amdgpu_kernel void @constant_load_v32i1(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: constant_load_v32i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -680,6 +704,8 @@ define amdgpu_kernel void @constant_load_v64i1(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: constant_load_v64i1:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -753,6 +779,8 @@ define amdgpu_kernel void @constant_zextload_i1_to_i32(ptr addrspace(1) %out, pt
;
; GFX1250-LABEL: constant_zextload_i1_to_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -831,6 +859,8 @@ define amdgpu_kernel void @constant_sextload_i1_to_i32(ptr addrspace(1) %out, pt
;
; GFX1250-LABEL: constant_sextload_i1_to_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -906,6 +936,8 @@ define amdgpu_kernel void @constant_zextload_v1i1_to_v1i32(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_zextload_v1i1_to_v1i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -984,6 +1016,8 @@ define amdgpu_kernel void @constant_sextload_v1i1_to_v1i32(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v1i1_to_v1i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1070,6 +1104,8 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i32(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_zextload_v2i1_to_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -1157,6 +1193,8 @@ define amdgpu_kernel void @constant_sextload_v2i1_to_v2i32(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v2i1_to_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1253,6 +1291,8 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i32(ptr addrspace(1) %out
;
; GFX1250-FAKE16-LABEL: constant_zextload_v3i1_to_v3i32:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, 0
@@ -1269,6 +1309,8 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i32(ptr addrspace(1) %out
;
; GFX1250-REAL16-LABEL: constant_zextload_v3i1_to_v3i32:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
@@ -1369,6 +1411,8 @@ define amdgpu_kernel void @constant_sextload_v3i1_to_v3i32(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v3i1_to_v3i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1467,6 +1511,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i32(ptr addrspace(1) %out
;
; GFX1250-FAKE16-LABEL: constant_zextload_v4i1_to_v4i32:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v4, 0
@@ -1484,6 +1530,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i32(ptr addrspace(1) %out
;
; GFX1250-REAL16-LABEL: constant_zextload_v4i1_to_v4i32:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, 0
@@ -1589,6 +1637,8 @@ define amdgpu_kernel void @constant_sextload_v4i1_to_v4i32(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v4i1_to_v4i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1729,6 +1779,8 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
;
; GFX1250-FAKE16-LABEL: constant_zextload_v8i1_to_v8i32:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v8, 0
@@ -1755,6 +1807,8 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
;
; GFX1250-REAL16-LABEL: constant_zextload_v8i1_to_v8i32:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v8, 0
@@ -1903,6 +1957,8 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i32(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v8i1_to_v8i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2121,6 +2177,8 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_zextload_v16i1_to_v16i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v16, 0
@@ -2351,6 +2409,8 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i32(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2742,6 +2802,8 @@ define amdgpu_kernel void @constant_zextload_v32i1_to_v32i32(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_zextload_v32i1_to_v32i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3200,6 +3262,8 @@ define amdgpu_kernel void @constant_sextload_v32i1_to_v32i32(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_sextload_v32i1_to_v32i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3939,6 +4003,8 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i32(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_zextload_v64i1_to_v64i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4810,6 +4876,8 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i32(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_sextload_v64i1_to_v64i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5015,6 +5083,8 @@ define amdgpu_kernel void @constant_zextload_i1_to_i64(ptr addrspace(1) %out, pt
;
; GFX1250-LABEL: constant_zextload_i1_to_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5099,6 +5169,8 @@ define amdgpu_kernel void @constant_sextload_i1_to_i64(ptr addrspace(1) %out, pt
;
; GFX1250-LABEL: constant_sextload_i1_to_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -5182,6 +5254,8 @@ define amdgpu_kernel void @constant_zextload_v1i1_to_v1i64(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_zextload_v1i1_to_v1i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5266,6 +5340,8 @@ define amdgpu_kernel void @constant_sextload_v1i1_to_v1i64(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v1i1_to_v1i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -5364,6 +5440,8 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
;
; GFX1250-FAKE16-LABEL: constant_zextload_v2i1_to_v2i64:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -5382,6 +5460,8 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
;
; GFX1250-REAL16-LABEL: constant_zextload_v2i1_to_v2i64:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -5487,6 +5567,8 @@ define amdgpu_kernel void @constant_sextload_v2i1_to_v2i64(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v2i1_to_v2i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
@@ -5606,6 +5688,8 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
;
; GFX1250-FAKE16-LABEL: constant_zextload_v3i1_to_v3i64:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, 0
@@ -5630,6 +5714,8 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
;
; GFX1250-REAL16-LABEL: constant_zextload_v3i1_to_v3i64:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -5765,6 +5851,8 @@ define amdgpu_kernel void @constant_sextload_v3i1_to_v3i64(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v3i1_to_v3i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v5, 0
@@ -5903,6 +5991,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
;
; GFX1250-FAKE16-LABEL: constant_zextload_v4i1_to_v4i64:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -5932,6 +6022,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
;
; GFX1250-REAL16-LABEL: constant_zextload_v4i1_to_v4i64:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
@@ -6080,6 +6172,8 @@ define amdgpu_kernel void @constant_sextload_v4i1_to_v4i64(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v4i1_to_v4i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v9, 0
@@ -6265,6 +6359,8 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i64(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_zextload_v8i1_to_v8i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -6498,6 +6594,8 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
;
; GFX1250-LABEL: constant_sextload_v8i1_to_v8i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v16, 0
@@ -6820,6 +6918,8 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
;
; GFX1250-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
@@ -6864,6 +6964,8 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
;
; GFX1250-REAL16-LABEL: constant_zextload_v16i1_to_v16i64:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
@@ -7275,6 +7377,8 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v32, 0
@@ -7850,6 +7954,8 @@ define amdgpu_kernel void @constant_zextload_v32i1_to_v32i64(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_zextload_v32i1_to_v32i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8626,6 +8732,8 @@ define amdgpu_kernel void @constant_sextload_v32i1_to_v32i64(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_sextload_v32i1_to_v32i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9742,6 +9850,8 @@ define amdgpu_kernel void @constant_zextload_v64i1_to_v64i64(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_zextload_v64i1_to_v64i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11305,6 +11415,8 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
;
; GFX1250-LABEL: constant_sextload_v64i1_to_v64i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s61, 0
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index 7770149c5cde9..4b374eb86c451 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -87,6 +87,8 @@ define amdgpu_kernel void @constant_load_i32(ptr addrspace(1) %out, ptr addrspac
;
; GFX1250-LABEL: constant_load_i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -185,6 +187,8 @@ define amdgpu_kernel void @constant_load_v2i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: constant_load_v2i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -292,6 +296,8 @@ define amdgpu_kernel void @constant_load_v3i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: constant_load_v3i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -400,6 +406,8 @@ define amdgpu_kernel void @constant_load_v4i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: constant_load_v4i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
@@ -544,6 +552,8 @@ define amdgpu_kernel void @constant_load_v8i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: constant_load_v8i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -723,6 +733,8 @@ define amdgpu_kernel void @constant_load_v9i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: constant_load_v9i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -910,6 +922,8 @@ define amdgpu_kernel void @constant_load_v10i32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: constant_load_v10i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1106,6 +1120,8 @@ define amdgpu_kernel void @constant_load_v11i32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: constant_load_v11i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1305,6 +1321,8 @@ define amdgpu_kernel void @constant_load_v12i32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: constant_load_v12i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1530,6 +1548,8 @@ define amdgpu_kernel void @constant_load_v16i32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: constant_load_v16i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[16:19], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1639,6 +1659,8 @@ define amdgpu_kernel void @constant_zextload_i32_to_i64(ptr addrspace(1) %out, p
;
; GFX1250-LABEL: constant_zextload_i32_to_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1744,6 +1766,8 @@ define amdgpu_kernel void @constant_sextload_i32_to_i64(ptr addrspace(1) %out, p
;
; GFX1250-LABEL: constant_sextload_i32_to_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1844,6 +1868,8 @@ define amdgpu_kernel void @constant_zextload_v1i32_to_v1i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_zextload_v1i32_to_v1i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1949,6 +1975,8 @@ define amdgpu_kernel void @constant_sextload_v1i32_to_v1i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_sextload_v1i32_to_v1i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2064,6 +2092,8 @@ define amdgpu_kernel void @constant_zextload_v2i32_to_v2i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_zextload_v2i32_to_v2i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2190,6 +2220,8 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_sextload_v2i32_to_v2i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2338,6 +2370,8 @@ define amdgpu_kernel void @constant_zextload_v4i32_to_v4i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_zextload_v4i32_to_v4i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2516,6 +2550,8 @@ define amdgpu_kernel void @constant_sextload_v4i32_to_v4i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_sextload_v4i32_to_v4i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2735,6 +2771,8 @@ define amdgpu_kernel void @constant_zextload_v8i32_to_v8i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_zextload_v8i32_to_v8i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3025,6 +3063,8 @@ define amdgpu_kernel void @constant_sextload_v8i32_to_v8i64(ptr addrspace(1) %ou
;
; GFX1250-LABEL: constant_sextload_v8i32_to_v8i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3527,6 +3567,8 @@ define amdgpu_kernel void @constant_sextload_v16i32_to_v16i64(ptr addrspace(1) %
;
; GFX1250-LABEL: constant_sextload_v16i32_to_v16i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[16:19], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3916,6 +3958,8 @@ define amdgpu_kernel void @constant_zextload_v16i32_to_v16i64(ptr addrspace(1) %
;
; GFX1250-LABEL: constant_zextload_v16i32_to_v16i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[16:19], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4834,6 +4878,8 @@ define amdgpu_kernel void @constant_sextload_v32i32_to_v32i64(ptr addrspace(1) %
;
; GFX1250-LABEL: constant_sextload_v32i32_to_v32i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[36:39], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5550,6 +5596,8 @@ define amdgpu_kernel void @constant_zextload_v32i32_to_v32i64(ptr addrspace(1) %
;
; GFX1250-LABEL: constant_zextload_v32i32_to_v32i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[36:39], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5972,6 +6020,8 @@ define amdgpu_kernel void @constant_load_v32i32(ptr addrspace(1) %out, ptr addrs
;
; GFX1250-LABEL: constant_load_v32i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[36:39], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll b/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll
index 8a505cd5185d2..513ca7985c55b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll
@@ -18,6 +18,8 @@ define amdgpu_ps <2 x float> @global_load_scale_add_foldable_knownbits(ptr addrs
;
; GFX1250-LABEL: global_load_scale_add_foldable_knownbits:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[2:3] offset:128 scale_offset
@@ -43,6 +45,8 @@ define amdgpu_ps <2 x float> @global_load_scale_add_foldable_nowrap(ptr addrspac
;
; GFX1250-LABEL: global_load_scale_add_foldable_nowrap:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[2:3] offset:128 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -66,6 +70,8 @@ define amdgpu_ps <2 x float> @global_load_scale_add_unfoldable(ptr addrspace(1)
;
; GFX1250-LABEL: global_load_scale_add_unfoldable:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshl_add_u32 v0, v0, 3, 0x80
; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
index 199b79932402f..87f1b243b8fb3 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll
@@ -101,6 +101,8 @@ define amdgpu_kernel void @copy_flat(ptr nocapture %d, ptr nocapture readonly %s
;
; GFX1250-LABEL: copy_flat:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -224,6 +226,8 @@ define amdgpu_kernel void @copy_global(ptr addrspace(1) nocapture %d, ptr addrsp
;
; GFX1250-LABEL: copy_global:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -351,6 +355,8 @@ define amdgpu_kernel void @copy_constant(ptr addrspace(1) nocapture %d, ptr addr
;
; GFX1250-LABEL: copy_constant:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -478,6 +484,8 @@ define amdgpu_kernel void @copy_local(ptr addrspace(3) nocapture %d, ptr addrspa
;
; GFX1250-LABEL: copy_local:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -650,6 +658,8 @@ define amdgpu_kernel void @copy_flat_divergent(ptr nocapture %d, ptr nocapture r
;
; GFX1250-LABEL: copy_flat_divergent:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x34 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -826,6 +836,8 @@ define amdgpu_kernel void @copy_global_divergent(ptr addrspace(1) nocapture %d,
;
; GFX1250-LABEL: copy_global_divergent:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x34 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics.ll
index 81d2b542aa907..8243f44b4a2eb 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics.ll
@@ -49,6 +49,8 @@ define amdgpu_cs void @_amdgpu_cs_main() {
;
; GFX1250-SDAG-LABEL: _amdgpu_cs_main:
; GFX1250-SDAG: ; %bb.0: ; %.entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_bfe_u32 s0, ttmp6, 0x4000c
; GFX1250-SDAG-NEXT: s_bfe_u32 s2, ttmp6, 0x40010
@@ -80,6 +82,8 @@ define amdgpu_cs void @_amdgpu_cs_main() {
;
; GFX1250-GISEL-LABEL: _amdgpu_cs_main:
; GFX1250-GISEL: ; %bb.0: ; %.entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_bfe_u32 s0, ttmp6, 0x4000c
; GFX1250-GISEL-NEXT: s_and_b32 s1, ttmp6, 15
@@ -166,6 +170,8 @@ define amdgpu_cs void @workgroup_id_no_clusters() "amdgpu-cluster-dims"="0,0,0"
;
; GFX1250-SDAG-LABEL: workgroup_id_no_clusters:
; GFX1250-SDAG: ; %bb.0: ; %.entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_and_b32 s0, ttmp7, 0xffff
; GFX1250-SDAG-NEXT: s_lshr_b32 s1, ttmp7, 16
@@ -176,6 +182,8 @@ define amdgpu_cs void @workgroup_id_no_clusters() "amdgpu-cluster-dims"="0,0,0"
;
; GFX1250-GISEL-LABEL: workgroup_id_no_clusters:
; GFX1250-GISEL: ; %bb.0: ; %.entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_mov_b32 s0, ttmp9
; GFX1250-GISEL-NEXT: s_and_b32 s1, ttmp7, 0xffff
@@ -238,6 +246,8 @@ define amdgpu_cs void @workgroup_id_optimized() "amdgpu-cluster-dims"="2,3,4" {
;
; GFX1250-SDAG-LABEL: workgroup_id_optimized:
; GFX1250-SDAG: ; %bb.0: ; %.entry
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_lshr_b32 s1, ttmp7, 14
; GFX1250-SDAG-NEXT: s_and_b32 s2, ttmp7, 0xffff
@@ -257,6 +267,8 @@ define amdgpu_cs void @workgroup_id_optimized() "amdgpu-cluster-dims"="2,3,4" {
;
; GFX1250-GISEL-LABEL: workgroup_id_optimized:
; GFX1250-GISEL: ; %bb.0: ; %.entry
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_and_b32 s1, ttmp7, 0xffff
; GFX1250-GISEL-NEXT: s_and_b32 s0, ttmp6, 15
@@ -339,6 +351,8 @@ define amdgpu_cs void @caller() {
;
; GFX1250-SDAG-LABEL: caller:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_mov_b32 s32, 0
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_bfe_u32 s0, ttmp6, 0x4000c
@@ -357,6 +371,8 @@ define amdgpu_cs void @caller() {
;
; GFX1250-GISEL-LABEL: caller:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_mov_b32 s32, 0
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_bfe_u32 s0, ttmp6, 0x4000c
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
index e382f637a25cf..07b0763d9d81e 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
@@ -604,6 +604,8 @@ define float @v_mad_mix_f32_bf16lo_cast_from_half_bf16lo_bf16lo(half %src0, bflo
define amdgpu_kernel void @test_fma_mix_f32_bf16_src2_bf16lo(float %x, i32 %y, ptr addrspace(1) %out) {
; GFX1250-LABEL: test_fma_mix_f32_bf16_src2_bf16lo:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 567f80f9f576b..b71fc6550d547 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -1119,6 +1119,8 @@ define amdgpu_kernel void @mad_i64_i32_uniform(ptr addrspace(1) %out, i32 %arg0,
;
; GFX1250-LABEL: mad_i64_i32_uniform:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -2228,6 +2230,8 @@ define amdgpu_ps i64 @lshr_mad_i64_sgpr(i64 inreg %arg0) #0 {
;
; GFX1250-LABEL: lshr_mad_i64_sgpr:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s3, 0
; GFX1250-NEXT: s_mov_b32 s2, s1
diff --git a/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll b/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll
index 577fe2f54e551..3cafc3bec43c0 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll
@@ -26,6 +26,8 @@ define amdgpu_ps float @mad_i32_vvv(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: mad_i32_vvv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
@@ -59,6 +61,8 @@ define amdgpu_ps float @mad_i32_sss(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
;
; GFX1250-LABEL: mad_i32_sss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mul_i32 s0, s0, s1
; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
@@ -90,6 +94,8 @@ define amdgpu_ps float @mad_i32_vvc(i32 %a, i32 %b) {
;
; GFX1250-LABEL: mad_i32_vvc:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, v1, 42
; GFX1250-NEXT: ; return to shader part epilog
@@ -121,6 +127,8 @@ define amdgpu_ps float @mad_i32_vvi(i32 %a, i32 %b) {
;
; GFX1250-LABEL: mad_i32_vvi:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, v1, 0x12d687
; GFX1250-NEXT: ; return to shader part epilog
@@ -152,6 +160,8 @@ define amdgpu_ps float @mad_i32_vvi_neg(i32 %a, i32 %b) {
;
; GFX1250-LABEL: mad_i32_vvi_neg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, v1, 0xffed2979
; GFX1250-NEXT: ; return to shader part epilog
@@ -180,6 +190,8 @@ define amdgpu_ps float @mad_i32_vcv(i32 %a, i32 %c) {
;
; GFX1250-LABEL: mad_i32_vcv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, 42, v1
; GFX1250-NEXT: ; return to shader part epilog
@@ -208,6 +220,8 @@ define amdgpu_ps float @mad_i32_vcc(i32 %a) {
;
; GFX1250-LABEL: mad_i32_vcc:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, 42, 43
; GFX1250-NEXT: ; return to shader part epilog
@@ -237,6 +251,8 @@ define amdgpu_ps float @mad_i32_vvs(i32 %a, i32 %b, i32 inreg %c) {
;
; GFX1250-LABEL: mad_i32_vvs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, v1, s0
; GFX1250-NEXT: ; return to shader part epilog
@@ -265,6 +281,8 @@ define amdgpu_ps float @mad_i32_vsv(i32 %a, i32 inreg %b, i32 %c) {
;
; GFX1250-LABEL: mad_i32_vsv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, s0, v1
; GFX1250-NEXT: ; return to shader part epilog
@@ -293,6 +311,8 @@ define amdgpu_ps float @mad_i32_svv(i32 inreg %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: mad_i32_svv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, s0, v0, v1
; GFX1250-NEXT: ; return to shader part epilog
@@ -324,6 +344,8 @@ define amdgpu_ps float @mad_i32_vss(i32 %a, i32 inreg %b, i32 inreg %c) {
;
; GFX1250-LABEL: mad_i32_vss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, v0, s0, s1
; GFX1250-NEXT: ; return to shader part epilog
@@ -355,6 +377,8 @@ define amdgpu_ps float @mad_i32_svs(i32 inreg %a, i32 %b, i32 inreg %c) {
;
; GFX1250-LABEL: mad_i32_svs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, s0, v0, s1
; GFX1250-NEXT: ; return to shader part epilog
@@ -384,6 +408,8 @@ define amdgpu_ps float @mad_i32_ssv(i32 inreg %a, i32 inreg %b, i32 %c) {
;
; GFX1250-LABEL: mad_i32_ssv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mad_u32 v0, s0, s1, v0
; GFX1250-NEXT: ; return to shader part epilog
@@ -420,6 +446,8 @@ define amdgpu_ps float @mad_i32_vvv_multiuse(i32 %a, i32 %b, i32 %c) {
;
; GFX1250-LABEL: mad_i32_vvv_multiuse:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mul_lo_u32 v1, v0, v1
; GFX1250-NEXT: v_add_nc_u32_e32 v0, v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 84ef83d47a8db..105441cd35390 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -27,6 +27,8 @@ define amdgpu_kernel void @v_test_imax_sge_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_imax_sge_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -101,6 +103,8 @@ define amdgpu_kernel void @v_test_imax_sge_v4i32(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_imax_sge_v4i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -168,6 +172,8 @@ define amdgpu_kernel void @s_test_imax_sge_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_imax_sge_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -209,6 +215,8 @@ define amdgpu_kernel void @s_test_imax_sge_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX1250-LABEL: s_test_imax_sge_imm_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -260,6 +268,8 @@ define amdgpu_kernel void @v_test_imax_sge_i8(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-LABEL: v_test_imax_sge_i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -332,6 +342,8 @@ define amdgpu_kernel void @s_test_imax_sgt_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX1250-LABEL: s_test_imax_sgt_imm_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -375,6 +387,8 @@ define amdgpu_kernel void @s_test_imax_sgt_imm_v2i32(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: s_test_imax_sgt_imm_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -428,6 +442,8 @@ define amdgpu_kernel void @v_test_imax_sgt_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_imax_sgt_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -489,6 +505,8 @@ define amdgpu_kernel void @s_test_imax_sgt_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_imax_sgt_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -539,6 +557,8 @@ define amdgpu_kernel void @v_test_umax_uge_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_umax_uge_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -600,6 +620,8 @@ define amdgpu_kernel void @s_test_umax_uge_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_umax_uge_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -647,6 +669,8 @@ define amdgpu_kernel void @s_test_umax_uge_v3i32(ptr addrspace(1) %out, <3 x i32
;
; GFX1250-LABEL: s_test_umax_uge_v3i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -707,6 +731,8 @@ define amdgpu_kernel void @v_test_umax_uge_i8(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-LABEL: v_test_umax_uge_i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -778,6 +804,8 @@ define amdgpu_kernel void @v_test_umax_ugt_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_umax_ugt_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -837,6 +865,8 @@ define amdgpu_kernel void @s_test_umax_ugt_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_umax_ugt_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -880,6 +910,8 @@ define amdgpu_kernel void @s_test_umax_ugt_imm_v2i32(ptr addrspace(1) %out, <2 x
;
; GFX1250-LABEL: s_test_umax_ugt_imm_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -929,6 +961,8 @@ define amdgpu_kernel void @simplify_demanded_bits_test_umax_ugt_i16(ptr addrspac
;
; GFX1250-LABEL: simplify_demanded_bits_test_umax_ugt_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
@@ -993,6 +1027,8 @@ define amdgpu_kernel void @simplify_demanded_bits_test_max_slt_i16(ptr addrspace
;
; GFX1250-LABEL: simplify_demanded_bits_test_max_slt_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
@@ -1056,6 +1092,8 @@ define amdgpu_kernel void @s_test_imax_sge_i16(ptr addrspace(1) %out, [8 x i32],
;
; GFX1250-LABEL: s_test_imax_sge_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x70 nv
@@ -1131,6 +1169,8 @@ define amdgpu_kernel void @test_umax_ugt_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_umax_ugt_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1185,6 +1225,8 @@ define amdgpu_kernel void @test_umax_uge_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_umax_uge_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1239,6 +1281,8 @@ define amdgpu_kernel void @test_imax_sgt_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_imax_sgt_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -1293,6 +1337,8 @@ define amdgpu_kernel void @test_imax_sge_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_imax_sge_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-barriers.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-barriers.ll
index a325083045949..2a04466bee0a3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-barriers.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-barriers.ll
@@ -42,6 +42,8 @@ define amdgpu_kernel void @test_s_barrier() {
;
; GFX1250-LABEL: test_s_barrier:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_barrier_signal -1
; GFX1250-NEXT: s_barrier_wait -1
@@ -102,6 +104,8 @@ define amdgpu_kernel void @test_s_barrier_workgroup_fence() {
;
; GFX1250-LABEL: test_s_barrier_workgroup_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -165,6 +169,8 @@ define amdgpu_kernel void @test_s_barrier_agent_fence() {
;
; GFX1250-LABEL: test_s_barrier_agent_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll
index 9e977821b191d..5e9289722ff76 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll
@@ -82,6 +82,8 @@ define amdgpu_kernel void @workgroup_acquire_fence() #0 {
;
; GFX1250-LABEL: workgroup_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -164,6 +166,8 @@ define amdgpu_kernel void @workgroup_release_fence() #0 {
;
; GFX1250-LABEL: workgroup_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -251,6 +255,8 @@ define amdgpu_kernel void @workgroup_acq_rel_fence() #0 {
;
; GFX1250-LABEL: workgroup_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -338,6 +344,8 @@ define amdgpu_kernel void @workgroup_seq_cst_fence() #0 {
;
; GFX1250-LABEL: workgroup_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -415,6 +423,8 @@ define amdgpu_kernel void @workgroup_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -497,6 +507,8 @@ define amdgpu_kernel void @workgroup_one_as_release_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -584,6 +596,8 @@ define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -671,6 +685,8 @@ define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -770,6 +786,8 @@ define amdgpu_kernel void @agent_acquire_fence() #0 {
;
; GFX1250-LABEL: agent_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -861,6 +879,8 @@ define amdgpu_kernel void @agent_release_fence() #0 {
;
; GFX1250-LABEL: agent_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -969,6 +989,8 @@ define amdgpu_kernel void @agent_acq_rel_fence() #0 {
;
; GFX1250-LABEL: agent_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1079,6 +1101,8 @@ define amdgpu_kernel void @agent_seq_cst_fence() #0 {
;
; GFX1250-LABEL: agent_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1183,6 +1207,8 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1274,6 +1300,8 @@ define amdgpu_kernel void @agent_one_as_release_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1382,6 +1410,8 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1492,6 +1522,8 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1598,6 +1630,8 @@ define amdgpu_kernel void @system_acquire_fence() #0 {
;
; GFX1250-LABEL: system_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1693,6 +1727,8 @@ define amdgpu_kernel void @system_release_fence() #0 {
;
; GFX1250-LABEL: system_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1807,6 +1843,8 @@ define amdgpu_kernel void @system_acq_rel_fence() #0 {
;
; GFX1250-LABEL: system_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1923,6 +1961,8 @@ define amdgpu_kernel void @system_seq_cst_fence() #0 {
;
; GFX1250-LABEL: system_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2029,6 +2069,8 @@ define amdgpu_kernel void @system_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: system_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2124,6 +2166,8 @@ define amdgpu_kernel void @system_one_as_release_fence() #0 {
;
; GFX1250-LABEL: system_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2238,6 +2282,8 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: system_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2354,6 +2400,8 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: system_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll
index 83d12822ebce9..76b6acb113e7e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll
@@ -80,6 +80,8 @@ define amdgpu_kernel void @workgroup_acquire_fence() #0 {
;
; GFX1250-LABEL: workgroup_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -154,6 +156,8 @@ define amdgpu_kernel void @workgroup_release_fence() #0 {
;
; GFX1250-LABEL: workgroup_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -228,6 +232,8 @@ define amdgpu_kernel void @workgroup_acq_rel_fence() #0 {
;
; GFX1250-LABEL: workgroup_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -302,6 +308,8 @@ define amdgpu_kernel void @workgroup_seq_cst_fence() #0 {
;
; GFX1250-LABEL: workgroup_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -365,6 +373,8 @@ define amdgpu_kernel void @workgroup_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -427,6 +437,8 @@ define amdgpu_kernel void @workgroup_one_as_release_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -489,6 +501,8 @@ define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -551,6 +565,8 @@ define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -624,6 +640,8 @@ define amdgpu_kernel void @agent_acquire_fence() #0 {
;
; GFX1250-LABEL: agent_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -698,6 +716,8 @@ define amdgpu_kernel void @agent_release_fence() #0 {
;
; GFX1250-LABEL: agent_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -772,6 +792,8 @@ define amdgpu_kernel void @agent_acq_rel_fence() #0 {
;
; GFX1250-LABEL: agent_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -846,6 +868,8 @@ define amdgpu_kernel void @agent_seq_cst_fence() #0 {
;
; GFX1250-LABEL: agent_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -909,6 +933,8 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -971,6 +997,8 @@ define amdgpu_kernel void @agent_one_as_release_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -1033,6 +1061,8 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -1095,6 +1125,8 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -1168,6 +1200,8 @@ define amdgpu_kernel void @system_acquire_fence() #0 {
;
; GFX1250-LABEL: system_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -1242,6 +1276,8 @@ define amdgpu_kernel void @system_release_fence() #0 {
;
; GFX1250-LABEL: system_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -1316,6 +1352,8 @@ define amdgpu_kernel void @system_acq_rel_fence() #0 {
;
; GFX1250-LABEL: system_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -1390,6 +1428,8 @@ define amdgpu_kernel void @system_seq_cst_fence() #0 {
;
; GFX1250-LABEL: system_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -1453,6 +1493,8 @@ define amdgpu_kernel void @system_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: system_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -1515,6 +1557,8 @@ define amdgpu_kernel void @system_one_as_release_fence() #0 {
;
; GFX1250-LABEL: system_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -1577,6 +1621,8 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: system_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -1639,6 +1685,8 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: system_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
index 24252d05f019e..c4eda93a0a81d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
@@ -69,6 +69,8 @@ define amdgpu_kernel void @singlethread_acquire_fence() #0 {
;
; GFX1250-LABEL: singlethread_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -131,6 +133,8 @@ define amdgpu_kernel void @singlethread_release_fence() #0 {
;
; GFX1250-LABEL: singlethread_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -193,6 +197,8 @@ define amdgpu_kernel void @singlethread_acq_rel_fence() #0 {
;
; GFX1250-LABEL: singlethread_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -255,6 +261,8 @@ define amdgpu_kernel void @singlethread_seq_cst_fence() #0 {
;
; GFX1250-LABEL: singlethread_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -317,6 +325,8 @@ define amdgpu_kernel void @singlethread_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: singlethread_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -379,6 +389,8 @@ define amdgpu_kernel void @singlethread_one_as_release_fence() #0 {
;
; GFX1250-LABEL: singlethread_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -441,6 +453,8 @@ define amdgpu_kernel void @singlethread_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: singlethread_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -503,6 +517,8 @@ define amdgpu_kernel void @singlethread_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: singlethread_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -565,6 +581,8 @@ define amdgpu_kernel void @wavefront_acquire_fence() #0 {
;
; GFX1250-LABEL: wavefront_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -627,6 +645,8 @@ define amdgpu_kernel void @wavefront_release_fence() #0 {
;
; GFX1250-LABEL: wavefront_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -689,6 +709,8 @@ define amdgpu_kernel void @wavefront_acq_rel_fence() #0 {
;
; GFX1250-LABEL: wavefront_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -751,6 +773,8 @@ define amdgpu_kernel void @wavefront_seq_cst_fence() #0 {
;
; GFX1250-LABEL: wavefront_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -813,6 +837,8 @@ define amdgpu_kernel void @wavefront_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: wavefront_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -875,6 +901,8 @@ define amdgpu_kernel void @wavefront_one_as_release_fence() #0 {
;
; GFX1250-LABEL: wavefront_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -937,6 +965,8 @@ define amdgpu_kernel void @wavefront_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: wavefront_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -999,6 +1029,8 @@ define amdgpu_kernel void @wavefront_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: wavefront_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_endpgm
entry:
@@ -1082,6 +1114,8 @@ define amdgpu_kernel void @workgroup_acquire_fence() #0 {
;
; GFX1250-LABEL: workgroup_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1169,6 +1203,8 @@ define amdgpu_kernel void @workgroup_release_fence() #0 {
;
; GFX1250-LABEL: workgroup_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1261,6 +1297,8 @@ define amdgpu_kernel void @workgroup_acq_rel_fence() #0 {
;
; GFX1250-LABEL: workgroup_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1353,6 +1391,8 @@ define amdgpu_kernel void @workgroup_seq_cst_fence() #0 {
;
; GFX1250-LABEL: workgroup_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1430,6 +1470,8 @@ define amdgpu_kernel void @workgroup_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1512,6 +1554,8 @@ define amdgpu_kernel void @workgroup_one_as_release_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1599,6 +1643,8 @@ define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1686,6 +1732,8 @@ define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: workgroup_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -1786,6 +1834,8 @@ define amdgpu_kernel void @cluster_acquire_fence() #0 {
;
; GFX1250-LABEL: cluster_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1876,6 +1926,8 @@ define amdgpu_kernel void @cluster_release_fence() #0 {
;
; GFX1250-LABEL: cluster_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1981,6 +2033,8 @@ define amdgpu_kernel void @cluster_acq_rel_fence() #0 {
;
; GFX1250-LABEL: cluster_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2087,6 +2141,8 @@ define amdgpu_kernel void @cluster_seq_cst_fence() #0 {
;
; GFX1250-LABEL: cluster_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2187,6 +2243,8 @@ define amdgpu_kernel void @cluster_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: cluster_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2277,6 +2335,8 @@ define amdgpu_kernel void @cluster_one_as_release_fence() #0 {
;
; GFX1250-LABEL: cluster_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2382,6 +2442,8 @@ define amdgpu_kernel void @cluster_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: cluster_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2488,6 +2550,8 @@ define amdgpu_kernel void @cluster_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: cluster_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2588,6 +2652,8 @@ define amdgpu_kernel void @agent_acquire_fence() #0 {
;
; GFX1250-LABEL: agent_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -2679,6 +2745,8 @@ define amdgpu_kernel void @agent_release_fence() #0 {
;
; GFX1250-LABEL: agent_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2787,6 +2855,8 @@ define amdgpu_kernel void @agent_acq_rel_fence() #0 {
;
; GFX1250-LABEL: agent_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -2897,6 +2967,8 @@ define amdgpu_kernel void @agent_seq_cst_fence() #0 {
;
; GFX1250-LABEL: agent_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3001,6 +3073,8 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3092,6 +3166,8 @@ define amdgpu_kernel void @agent_one_as_release_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3200,6 +3276,8 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3310,6 +3388,8 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: agent_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3416,6 +3496,8 @@ define amdgpu_kernel void @system_acquire_fence() #0 {
;
; GFX1250-LABEL: system_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3511,6 +3593,8 @@ define amdgpu_kernel void @system_release_fence() #0 {
;
; GFX1250-LABEL: system_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3625,6 +3709,8 @@ define amdgpu_kernel void @system_acq_rel_fence() #0 {
;
; GFX1250-LABEL: system_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3741,6 +3827,8 @@ define amdgpu_kernel void @system_seq_cst_fence() #0 {
;
; GFX1250-LABEL: system_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3847,6 +3935,8 @@ define amdgpu_kernel void @system_one_as_acquire_fence() #0 {
;
; GFX1250-LABEL: system_one_as_acquire_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -3942,6 +4032,8 @@ define amdgpu_kernel void @system_one_as_release_fence() #0 {
;
; GFX1250-LABEL: system_one_as_release_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -4056,6 +4148,8 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() #0 {
;
; GFX1250-LABEL: system_one_as_acq_rel_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -4172,6 +4266,8 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() #0 {
;
; GFX1250-LABEL: system_one_as_seq_cst_fence:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_storecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
index c632cdb7b0820..7cd080dc05e98 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
@@ -239,6 +239,8 @@ define amdgpu_kernel void @flat_agent_unordered_load(
;
; GFX1250-LABEL: flat_agent_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -485,6 +487,8 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
;
; GFX1250-LABEL: flat_agent_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -738,6 +742,8 @@ define amdgpu_kernel void @flat_agent_acquire_load(
;
; GFX1250-LABEL: flat_agent_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1013,6 +1019,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
;
; GFX1250-LABEL: flat_agent_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1231,6 +1239,8 @@ define amdgpu_kernel void @flat_agent_unordered_store(
;
; GFX1250-LABEL: flat_agent_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1444,6 +1454,8 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
;
; GFX1250-LABEL: flat_agent_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1681,6 +1693,8 @@ define amdgpu_kernel void @flat_agent_release_store(
;
; GFX1250-LABEL: flat_agent_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1923,6 +1937,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
;
; GFX1250-LABEL: flat_agent_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2141,6 +2157,8 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_agent_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2385,6 +2403,8 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_agent_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2625,6 +2645,8 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
;
; GFX1250-LABEL: flat_agent_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2898,6 +2920,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_agent_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3174,6 +3198,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_agent_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3454,6 +3480,8 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_agent_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3755,6 +3783,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_agent_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4061,6 +4091,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_agent_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4396,6 +4428,8 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4759,6 +4793,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5118,6 +5154,8 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5510,6 +5548,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5905,6 +5945,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6276,6 +6318,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6642,6 +6686,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7032,6 +7078,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7427,6 +7475,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7822,6 +7872,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8217,6 +8269,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8612,6 +8666,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9007,6 +9063,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9402,6 +9460,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9797,6 +9857,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10181,6 +10243,8 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10576,6 +10640,8 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10982,6 +11048,8 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11406,6 +11474,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11832,6 +11902,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12234,6 +12306,8 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12631,6 +12705,8 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13052,6 +13128,8 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13478,6 +13556,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13904,6 +13984,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14330,6 +14412,8 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14756,6 +14840,8 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15182,6 +15268,8 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15608,6 +15696,8 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16034,6 +16124,8 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16297,6 +16389,8 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
;
; GFX1250-LABEL: flat_agent_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16543,6 +16637,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16805,6 +16901,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17091,6 +17189,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17309,6 +17409,8 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
;
; GFX1250-LABEL: flat_agent_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17522,6 +17624,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17759,6 +17863,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
;
; GFX1250-LABEL: flat_agent_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18001,6 +18107,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18219,6 +18327,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18459,6 +18569,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18699,6 +18811,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18968,6 +19082,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19240,6 +19356,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19530,6 +19648,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19841,6 +19961,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20157,6 +20279,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20492,6 +20616,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20851,6 +20977,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21210,6 +21338,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21598,6 +21728,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21989,6 +22121,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22356,6 +22490,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22718,6 +22854,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23104,6 +23242,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23495,6 +23635,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23886,6 +24028,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24277,6 +24421,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24668,6 +24814,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25059,6 +25207,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25450,6 +25600,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25841,6 +25993,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26225,6 +26379,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26630,6 +26786,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27036,6 +27194,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27470,6 +27630,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27906,6 +28068,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28318,6 +28482,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28725,6 +28891,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29156,6 +29324,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29592,6 +29762,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30028,6 +30200,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30464,6 +30638,8 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30900,6 +31076,8 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -31336,6 +31514,8 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -31772,6 +31952,8 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -32208,6 +32390,8 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-cluster.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-cluster.ll
index 03b14a3b75eb9..f7ba0609e7f17 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-cluster.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-cluster.ll
@@ -239,6 +239,8 @@ define amdgpu_kernel void @flat_cluster_unordered_load(
;
; GFX1250-LABEL: flat_cluster_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -485,6 +487,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_load(
;
; GFX1250-LABEL: flat_cluster_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -738,6 +742,8 @@ define amdgpu_kernel void @flat_cluster_acquire_load(
;
; GFX1250-LABEL: flat_cluster_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1012,6 +1018,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_load(
;
; GFX1250-LABEL: flat_cluster_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1229,6 +1237,8 @@ define amdgpu_kernel void @flat_cluster_unordered_store(
;
; GFX1250-LABEL: flat_cluster_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1442,6 +1452,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_store(
;
; GFX1250-LABEL: flat_cluster_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1679,6 +1691,8 @@ define amdgpu_kernel void @flat_cluster_release_store(
;
; GFX1250-LABEL: flat_cluster_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1918,6 +1932,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_store(
;
; GFX1250-LABEL: flat_cluster_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2133,6 +2149,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2377,6 +2395,8 @@ define amdgpu_kernel void @flat_cluster_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2616,6 +2636,8 @@ define amdgpu_kernel void @flat_cluster_release_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2886,6 +2908,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3158,6 +3182,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3434,6 +3460,8 @@ define amdgpu_kernel void @flat_cluster_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3734,6 +3762,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4036,6 +4066,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4367,6 +4399,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4730,6 +4764,8 @@ define amdgpu_kernel void @flat_cluster_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5088,6 +5124,8 @@ define amdgpu_kernel void @flat_cluster_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5477,6 +5515,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5868,6 +5908,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6235,6 +6277,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6600,6 +6644,8 @@ define amdgpu_kernel void @flat_cluster_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6989,6 +7035,8 @@ define amdgpu_kernel void @flat_cluster_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7380,6 +7428,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7771,6 +7821,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8162,6 +8214,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8553,6 +8607,8 @@ define amdgpu_kernel void @flat_cluster_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8944,6 +9000,8 @@ define amdgpu_kernel void @flat_cluster_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9335,6 +9393,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9726,6 +9786,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10106,6 +10168,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10501,6 +10565,8 @@ define amdgpu_kernel void @flat_cluster_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10906,6 +10972,8 @@ define amdgpu_kernel void @flat_cluster_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11327,6 +11395,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11749,6 +11819,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12147,6 +12219,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12543,6 +12617,8 @@ define amdgpu_kernel void @flat_cluster_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12963,6 +13039,8 @@ define amdgpu_kernel void @flat_cluster_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13385,6 +13463,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13807,6 +13887,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14229,6 +14311,8 @@ define amdgpu_kernel void @flat_cluster_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14651,6 +14735,8 @@ define amdgpu_kernel void @flat_cluster_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15073,6 +15159,8 @@ define amdgpu_kernel void @flat_cluster_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15495,6 +15583,8 @@ define amdgpu_kernel void @flat_cluster_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15917,6 +16007,8 @@ define amdgpu_kernel void @flat_cluster_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16176,6 +16268,8 @@ define amdgpu_kernel void @flat_cluster_one_as_unordered_load(
;
; GFX1250-LABEL: flat_cluster_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16422,6 +16516,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_load(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16684,6 +16780,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_load(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16970,6 +17068,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_load(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17188,6 +17288,8 @@ define amdgpu_kernel void @flat_cluster_one_as_unordered_store(
;
; GFX1250-LABEL: flat_cluster_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17401,6 +17503,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_store(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17638,6 +17742,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_store(
;
; GFX1250-LABEL: flat_cluster_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17877,6 +17983,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_store(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18092,6 +18200,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18332,6 +18442,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18571,6 +18683,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18837,6 +18951,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19105,6 +19221,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19391,6 +19509,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19702,6 +19822,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20015,6 +20137,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20347,6 +20471,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20706,6 +20832,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21064,6 +21192,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21449,6 +21579,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21836,6 +21968,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22199,6 +22333,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22560,6 +22696,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22945,6 +23083,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23332,6 +23472,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23719,6 +23861,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24106,6 +24250,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24493,6 +24639,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24880,6 +25028,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25267,6 +25417,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25654,6 +25806,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26034,6 +26188,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26439,6 +26595,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26845,6 +27003,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27276,6 +27436,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27709,6 +27871,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28118,6 +28282,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28525,6 +28691,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28956,6 +29124,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29389,6 +29559,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29822,6 +29994,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30255,6 +30429,8 @@ define amdgpu_kernel void @flat_cluster_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30688,6 +30864,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -31121,6 +31299,8 @@ define amdgpu_kernel void @flat_cluster_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -31554,6 +31734,8 @@ define amdgpu_kernel void @flat_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -31987,6 +32169,8 @@ define amdgpu_kernel void @flat_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
index d3c288b7cafa8..84810d44c992f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
@@ -26,6 +26,8 @@ define amdgpu_kernel void @flat_last_use_load_0(ptr %in, ptr %out) {
;
; GFX1250-LABEL: flat_last_use_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -97,6 +99,8 @@ define amdgpu_kernel void @flat_last_use_load_1(ptr %in, ptr %out) {
;
; GFX1250-LABEL: flat_last_use_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -164,6 +168,8 @@ define amdgpu_kernel void @flat_last_use_and_volatile_load(ptr %in, ptr %out) {
;
; GFX1250-LABEL: flat_last_use_and_volatile_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -223,6 +229,8 @@ define amdgpu_kernel void @flat_last_use_and_nontemporal_load(ptr %in, ptr %out)
;
; GFX1250-LABEL: flat_last_use_and_nontemporal_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 38ab005849ee5..c5a119b714fd4 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -239,6 +239,8 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
;
; GFX1250-LABEL: flat_nontemporal_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -639,6 +641,8 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
;
; GFX1250-LABEL: flat_nontemporal_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -891,6 +895,8 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
;
; GFX1250-LABEL: flat_nontemporal_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1287,6 +1293,8 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
;
; GFX1250-LABEL: flat_nontemporal_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1534,6 +1542,8 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
;
; GFX1250-LABEL: flat_nontemporal_volatile_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
index 15f5b6a5468ea..d9d5e24b66f3b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
@@ -239,6 +239,8 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
;
; GFX1250-LABEL: flat_singlethread_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -485,6 +487,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
;
; GFX1250-LABEL: flat_singlethread_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -731,6 +735,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
;
; GFX1250-LABEL: flat_singlethread_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -977,6 +983,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1191,6 +1199,8 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
;
; GFX1250-LABEL: flat_singlethread_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1404,6 +1414,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
;
; GFX1250-LABEL: flat_singlethread_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1617,6 +1629,8 @@ define amdgpu_kernel void @flat_singlethread_release_store(
;
; GFX1250-LABEL: flat_singlethread_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1830,6 +1844,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2043,6 +2059,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2256,6 +2274,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2469,6 +2489,8 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2682,6 +2704,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2895,6 +2919,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3152,6 +3178,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3412,6 +3440,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3672,6 +3702,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4000,6 +4032,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4332,6 +4366,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4664,6 +4700,8 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4996,6 +5034,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5328,6 +5368,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5660,6 +5702,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5992,6 +6036,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6324,6 +6370,8 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6656,6 +6704,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6988,6 +7038,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7320,6 +7372,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7652,6 +7706,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7984,6 +8040,8 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8316,6 +8374,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8648,6 +8708,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9024,6 +9086,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9404,6 +9468,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9784,6 +9850,8 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10164,6 +10232,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10544,6 +10614,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10924,6 +10996,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11304,6 +11378,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11684,6 +11760,8 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12064,6 +12142,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12444,6 +12524,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12824,6 +12906,8 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13204,6 +13288,8 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13584,6 +13670,8 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13964,6 +14052,8 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14344,6 +14434,8 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14600,6 +14692,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
;
; GFX1250-LABEL: flat_singlethread_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14846,6 +14940,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15092,6 +15188,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15338,6 +15436,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15552,6 +15652,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
;
; GFX1250-LABEL: flat_singlethread_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15765,6 +15867,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15978,6 +16082,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
;
; GFX1250-LABEL: flat_singlethread_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16191,6 +16297,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16404,6 +16512,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16617,6 +16727,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16830,6 +16942,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17043,6 +17157,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17256,6 +17372,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17513,6 +17631,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17773,6 +17893,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18033,6 +18155,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18361,6 +18485,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18693,6 +18819,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19025,6 +19153,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19357,6 +19487,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19689,6 +19821,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20021,6 +20155,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20353,6 +20489,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20685,6 +20823,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21017,6 +21157,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21349,6 +21491,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21681,6 +21825,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22013,6 +22159,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22345,6 +22493,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22677,6 +22827,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23009,6 +23161,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23385,6 +23539,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23765,6 +23921,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24145,6 +24303,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
;
; GFX1250-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24525,6 +24685,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24905,6 +25067,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25285,6 +25449,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25665,6 +25831,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26045,6 +26213,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26425,6 +26595,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26805,6 +26977,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27185,6 +27359,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
;
; GFX1250-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27565,6 +27741,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27945,6 +28123,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28325,6 +28505,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28705,6 +28887,8 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
index 4f02eb99086dd..43493aa690291 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
@@ -239,6 +239,8 @@ define amdgpu_kernel void @flat_system_unordered_load(
;
; GFX1250-LABEL: flat_system_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -485,6 +487,8 @@ define amdgpu_kernel void @flat_system_monotonic_load(
;
; GFX1250-LABEL: flat_system_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -740,6 +744,8 @@ define amdgpu_kernel void @flat_system_acquire_load(
;
; GFX1250-LABEL: flat_system_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1017,6 +1023,8 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
;
; GFX1250-LABEL: flat_system_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1235,6 +1243,8 @@ define amdgpu_kernel void @flat_system_unordered_store(
;
; GFX1250-LABEL: flat_system_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1448,6 +1458,8 @@ define amdgpu_kernel void @flat_system_monotonic_store(
;
; GFX1250-LABEL: flat_system_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1689,6 +1701,8 @@ define amdgpu_kernel void @flat_system_release_store(
;
; GFX1250-LABEL: flat_system_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1935,6 +1949,8 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
;
; GFX1250-LABEL: flat_system_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2153,6 +2169,8 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_system_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2399,6 +2417,8 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_system_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2643,6 +2663,8 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
;
; GFX1250-LABEL: flat_system_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2922,6 +2944,8 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_system_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3204,6 +3228,8 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_system_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3486,6 +3512,8 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_system_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3793,6 +3821,8 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_system_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4105,6 +4135,8 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_system_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4440,6 +4472,8 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4805,6 +4839,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5168,6 +5204,8 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5566,6 +5604,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5967,6 +6007,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6340,6 +6382,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6708,6 +6752,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7104,6 +7150,8 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7505,6 +7553,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7906,6 +7956,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8307,6 +8359,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8708,6 +8762,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9109,6 +9165,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9510,6 +9568,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9911,6 +9971,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10295,6 +10357,8 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10692,6 +10756,8 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11102,6 +11168,8 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11532,6 +11600,8 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11964,6 +12034,8 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12368,6 +12440,8 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12767,6 +12841,8 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13194,6 +13270,8 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13626,6 +13704,8 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14058,6 +14138,8 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14490,6 +14572,8 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14922,6 +15006,8 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15354,6 +15440,8 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15786,6 +15874,8 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16218,6 +16308,8 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16481,6 +16573,8 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
;
; GFX1250-LABEL: flat_system_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16727,6 +16821,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16991,6 +17087,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
;
; GFX1250-LABEL: flat_system_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17279,6 +17377,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17497,6 +17597,8 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
;
; GFX1250-LABEL: flat_system_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17710,6 +17812,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17951,6 +18055,8 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
;
; GFX1250-LABEL: flat_system_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18197,6 +18303,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18415,6 +18523,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18657,6 +18767,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18901,6 +19013,8 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19176,6 +19290,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19454,6 +19570,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19746,6 +19864,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20063,6 +20183,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20385,6 +20507,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20720,6 +20844,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21081,6 +21207,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21444,6 +21572,8 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21838,6 +21968,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22235,6 +22367,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22604,6 +22738,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22968,6 +23104,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23360,6 +23498,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23757,6 +23897,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24154,6 +24296,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24551,6 +24695,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24948,6 +25094,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25345,6 +25493,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25742,6 +25892,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26139,6 +26291,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26523,6 +26677,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26930,6 +27086,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27340,6 +27498,8 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27780,6 +27940,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28222,6 +28384,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28636,6 +28800,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29045,6 +29211,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29482,6 +29650,8 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29924,6 +30094,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30366,6 +30538,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30808,6 +30982,8 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -31250,6 +31426,8 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -31692,6 +31870,8 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -32134,6 +32314,8 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -32576,6 +32758,8 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
index a832c376bdfe6..a50df25216f7c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
@@ -167,6 +167,8 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
;
; GFX1250-LABEL: flat_nontemporal_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -464,6 +466,8 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
;
; GFX1250-LABEL: flat_nontemporal_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -665,6 +669,8 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
;
; GFX1250-LABEL: flat_nontemporal_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -967,6 +973,8 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
;
; GFX1250-LABEL: flat_nontemporal_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1154,6 +1162,8 @@ define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
;
; GFX1250-LABEL: flat_volatile_workgroup_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1326,6 +1336,8 @@ define amdgpu_kernel void @flat_volatile_workgroup_release_store(
;
; GFX1250-LABEL: flat_volatile_workgroup_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
index 776bf99716fa0..bd8e46e967dbb 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
@@ -239,6 +239,8 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
;
; GFX1250-LABEL: flat_wavefront_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -485,6 +487,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
;
; GFX1250-LABEL: flat_wavefront_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -731,6 +735,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
;
; GFX1250-LABEL: flat_wavefront_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -977,6 +983,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1191,6 +1199,8 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
;
; GFX1250-LABEL: flat_wavefront_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1404,6 +1414,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
;
; GFX1250-LABEL: flat_wavefront_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1617,6 +1629,8 @@ define amdgpu_kernel void @flat_wavefront_release_store(
;
; GFX1250-LABEL: flat_wavefront_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1830,6 +1844,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2043,6 +2059,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2256,6 +2274,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2469,6 +2489,8 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2682,6 +2704,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2895,6 +2919,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3152,6 +3178,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3412,6 +3440,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3672,6 +3702,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4000,6 +4032,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4332,6 +4366,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4664,6 +4700,8 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4996,6 +5034,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5328,6 +5368,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5660,6 +5702,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5992,6 +6036,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6324,6 +6370,8 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6656,6 +6704,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6988,6 +7038,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7320,6 +7372,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7652,6 +7706,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7984,6 +8040,8 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8316,6 +8374,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8648,6 +8708,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9024,6 +9086,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9404,6 +9468,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9784,6 +9850,8 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10164,6 +10232,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10544,6 +10614,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10924,6 +10996,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11304,6 +11378,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11684,6 +11760,8 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12064,6 +12142,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12444,6 +12524,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12824,6 +12906,8 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13204,6 +13288,8 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13584,6 +13670,8 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13964,6 +14052,8 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14344,6 +14434,8 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14600,6 +14692,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
;
; GFX1250-LABEL: flat_wavefront_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14846,6 +14940,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15092,6 +15188,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15338,6 +15436,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15552,6 +15652,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
;
; GFX1250-LABEL: flat_wavefront_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15765,6 +15867,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15978,6 +16082,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
;
; GFX1250-LABEL: flat_wavefront_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16191,6 +16297,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16404,6 +16512,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16617,6 +16727,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16830,6 +16942,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17043,6 +17157,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17256,6 +17372,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17513,6 +17631,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17773,6 +17893,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18033,6 +18155,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18361,6 +18485,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18693,6 +18819,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19025,6 +19153,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19357,6 +19487,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19689,6 +19821,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20021,6 +20155,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20353,6 +20489,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20685,6 +20823,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21017,6 +21157,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21349,6 +21491,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21681,6 +21825,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22013,6 +22159,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22345,6 +22493,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22677,6 +22827,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23009,6 +23161,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23385,6 +23539,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23765,6 +23921,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24145,6 +24303,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24525,6 +24685,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24905,6 +25067,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25285,6 +25449,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25665,6 +25831,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26045,6 +26213,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26425,6 +26595,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26805,6 +26977,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27185,6 +27359,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27565,6 +27741,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27945,6 +28123,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28325,6 +28505,8 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
index 611fd49b780d4..2f6b780561f0d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
@@ -239,6 +239,8 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
;
; GFX1250-LABEL: flat_workgroup_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -485,6 +487,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
;
; GFX1250-LABEL: flat_workgroup_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -735,6 +739,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
;
; GFX1250-LABEL: flat_workgroup_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1002,6 +1008,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1218,6 +1226,8 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
;
; GFX1250-LABEL: flat_workgroup_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1431,6 +1441,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
;
; GFX1250-LABEL: flat_workgroup_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1666,6 +1678,8 @@ define amdgpu_kernel void @flat_workgroup_release_store(
;
; GFX1250-LABEL: flat_workgroup_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1903,6 +1917,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2118,6 +2134,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2350,6 +2368,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2586,6 +2606,8 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2844,6 +2866,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3103,6 +3127,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3375,6 +3401,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3666,6 +3694,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3959,6 +3989,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4289,6 +4321,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4640,6 +4674,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4995,6 +5031,8 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5372,6 +5410,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5750,6 +5790,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6104,6 +6146,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6456,6 +6500,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6832,6 +6878,8 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7210,6 +7258,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7588,6 +7638,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7966,6 +8018,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8345,6 +8399,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8737,6 +8793,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9139,6 +9197,8 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9552,6 +9612,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9965,6 +10027,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10359,6 +10423,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10751,6 +10817,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11162,6 +11230,8 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11575,6 +11645,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11988,6 +12060,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12404,6 +12478,8 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12820,6 +12896,8 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13233,6 +13311,8 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13646,6 +13726,8 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14059,6 +14141,8 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14317,6 +14401,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
;
; GFX1250-LABEL: flat_workgroup_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14563,6 +14649,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14815,6 +14903,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15084,6 +15174,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15300,6 +15392,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
;
; GFX1250-LABEL: flat_workgroup_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15513,6 +15607,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15744,6 +15840,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15977,6 +16075,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16192,6 +16292,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16415,6 +16517,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16647,6 +16751,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16893,6 +16999,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17140,6 +17248,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17408,6 +17518,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17697,6 +17809,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17988,6 +18102,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18318,6 +18434,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18660,6 +18778,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19011,6 +19131,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19376,6 +19498,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19742,6 +19866,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20087,6 +20213,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20430,6 +20558,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20794,6 +20924,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21160,6 +21292,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21526,6 +21660,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21889,6 +22025,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22252,6 +22390,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22618,6 +22758,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22984,6 +23126,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23350,6 +23494,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23729,6 +23875,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24117,6 +24265,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24515,6 +24665,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24926,6 +25078,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25337,6 +25491,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25727,6 +25883,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26115,6 +26273,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26524,6 +26684,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26935,6 +27097,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27346,6 +27510,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27754,6 +27920,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28162,6 +28330,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28573,6 +28743,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28984,6 +29156,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29395,6 +29569,8 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
index 6ffcfd59ca1b2..ee882a306d688 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
@@ -236,6 +236,8 @@ define amdgpu_kernel void @global_agent_unordered_load(
;
; GFX1250-LABEL: global_agent_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -478,6 +480,8 @@ define amdgpu_kernel void @global_agent_monotonic_load(
;
; GFX1250-LABEL: global_agent_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -744,6 +748,8 @@ define amdgpu_kernel void @global_agent_acquire_load(
;
; GFX1250-LABEL: global_agent_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1026,6 +1032,8 @@ define amdgpu_kernel void @global_agent_seq_cst_load(
;
; GFX1250-LABEL: global_agent_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1253,6 +1261,8 @@ define amdgpu_kernel void @global_agent_unordered_store(
;
; GFX1250-LABEL: global_agent_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1475,6 +1485,8 @@ define amdgpu_kernel void @global_agent_monotonic_store(
;
; GFX1250-LABEL: global_agent_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1714,6 +1726,8 @@ define amdgpu_kernel void @global_agent_release_store(
;
; GFX1250-LABEL: global_agent_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1958,6 +1972,8 @@ define amdgpu_kernel void @global_agent_seq_cst_store(
;
; GFX1250-LABEL: global_agent_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2185,6 +2201,8 @@ define amdgpu_kernel void @global_agent_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_agent_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2436,6 +2454,8 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw(
;
; GFX1250-LABEL: global_agent_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2686,6 +2706,8 @@ define amdgpu_kernel void @global_agent_release_atomicrmw(
;
; GFX1250-LABEL: global_agent_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2967,6 +2989,8 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_agent_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3251,6 +3275,8 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_agent_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3525,6 +3551,8 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_agent_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3821,6 +3849,8 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_agent_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4122,6 +4152,8 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_agent_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4444,6 +4476,8 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4792,6 +4826,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5139,6 +5175,8 @@ define amdgpu_kernel void @global_agent_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5517,6 +5555,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5898,6 +5938,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6254,6 +6296,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6605,6 +6649,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6981,6 +7027,8 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7362,6 +7410,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7743,6 +7793,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8124,6 +8176,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8505,6 +8559,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8886,6 +8942,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9267,6 +9325,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9648,6 +9708,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10005,6 +10067,8 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10374,6 +10438,8 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10754,6 +10820,8 @@ define amdgpu_kernel void @global_agent_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11153,6 +11221,8 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11554,6 +11624,8 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11930,6 +12002,8 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12301,6 +12375,8 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12697,6 +12773,8 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13098,6 +13176,8 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13499,6 +13579,8 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13900,6 +13982,8 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14301,6 +14385,8 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14702,6 +14788,8 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15103,6 +15191,8 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15504,6 +15594,8 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15763,6 +15855,8 @@ define amdgpu_kernel void @global_agent_one_as_unordered_load(
;
; GFX1250-LABEL: global_agent_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16005,6 +16099,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_load(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16271,6 +16367,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load(
;
; GFX1250-LABEL: global_agent_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16562,6 +16660,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16789,6 +16889,8 @@ define amdgpu_kernel void @global_agent_one_as_unordered_store(
;
; GFX1250-LABEL: global_agent_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17011,6 +17113,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_store(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17250,6 +17354,8 @@ define amdgpu_kernel void @global_agent_one_as_release_store(
;
; GFX1250-LABEL: global_agent_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17494,6 +17600,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_store(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17721,6 +17829,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17972,6 +18082,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18222,6 +18334,8 @@ define amdgpu_kernel void @global_agent_one_as_release_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18503,6 +18617,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18787,6 +18903,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19061,6 +19179,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19357,6 +19477,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19658,6 +19780,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19980,6 +20104,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20328,6 +20454,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20675,6 +20803,8 @@ define amdgpu_kernel void @global_agent_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21053,6 +21183,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21434,6 +21566,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21790,6 +21924,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22141,6 +22277,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22517,6 +22655,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22898,6 +23038,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23279,6 +23421,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23660,6 +23804,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24041,6 +24187,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24422,6 +24570,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24803,6 +24953,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25184,6 +25336,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25541,6 +25695,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25910,6 +26066,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26306,6 +26464,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26707,6 +26867,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27083,6 +27245,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27454,6 +27618,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27850,6 +28016,8 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28251,6 +28419,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28652,6 +28822,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29053,6 +29225,8 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29454,6 +29628,8 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29855,6 +30031,8 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30256,6 +30434,8 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30657,6 +30837,8 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-cluster.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-cluster.ll
index 35d4b6c54194a..5f95d719c4454 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-cluster.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-cluster.ll
@@ -236,6 +236,8 @@ define amdgpu_kernel void @global_cluster_unordered_load(
;
; GFX1250-LABEL: global_cluster_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -478,6 +480,8 @@ define amdgpu_kernel void @global_cluster_monotonic_load(
;
; GFX1250-LABEL: global_cluster_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -744,6 +748,8 @@ define amdgpu_kernel void @global_cluster_acquire_load(
;
; GFX1250-LABEL: global_cluster_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1025,6 +1031,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_load(
;
; GFX1250-LABEL: global_cluster_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1251,6 +1259,8 @@ define amdgpu_kernel void @global_cluster_unordered_store(
;
; GFX1250-LABEL: global_cluster_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1473,6 +1483,8 @@ define amdgpu_kernel void @global_cluster_monotonic_store(
;
; GFX1250-LABEL: global_cluster_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1712,6 +1724,8 @@ define amdgpu_kernel void @global_cluster_release_store(
;
; GFX1250-LABEL: global_cluster_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1953,6 +1967,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_store(
;
; GFX1250-LABEL: global_cluster_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2177,6 +2193,8 @@ define amdgpu_kernel void @global_cluster_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_cluster_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2428,6 +2446,8 @@ define amdgpu_kernel void @global_cluster_acquire_atomicrmw(
;
; GFX1250-LABEL: global_cluster_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2677,6 +2697,8 @@ define amdgpu_kernel void @global_cluster_release_atomicrmw(
;
; GFX1250-LABEL: global_cluster_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2955,6 +2977,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_cluster_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3235,6 +3259,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_cluster_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3505,6 +3531,8 @@ define amdgpu_kernel void @global_cluster_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_cluster_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3800,6 +3828,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_cluster_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4097,6 +4127,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_cluster_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4415,6 +4447,8 @@ define amdgpu_kernel void @global_cluster_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4763,6 +4797,8 @@ define amdgpu_kernel void @global_cluster_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5109,6 +5145,8 @@ define amdgpu_kernel void @global_cluster_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5484,6 +5522,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5861,6 +5901,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6213,6 +6255,8 @@ define amdgpu_kernel void @global_cluster_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6563,6 +6607,8 @@ define amdgpu_kernel void @global_cluster_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6938,6 +6984,8 @@ define amdgpu_kernel void @global_cluster_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7315,6 +7363,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7692,6 +7742,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8069,6 +8121,8 @@ define amdgpu_kernel void @global_cluster_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8446,6 +8500,8 @@ define amdgpu_kernel void @global_cluster_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8823,6 +8879,8 @@ define amdgpu_kernel void @global_cluster_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9200,6 +9258,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9577,6 +9637,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9930,6 +9992,8 @@ define amdgpu_kernel void @global_cluster_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10299,6 +10363,8 @@ define amdgpu_kernel void @global_cluster_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10678,6 +10744,8 @@ define amdgpu_kernel void @global_cluster_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11074,6 +11142,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11471,6 +11541,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11843,6 +11915,8 @@ define amdgpu_kernel void @global_cluster_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12213,6 +12287,8 @@ define amdgpu_kernel void @global_cluster_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12608,6 +12684,8 @@ define amdgpu_kernel void @global_cluster_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13005,6 +13083,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13402,6 +13482,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13799,6 +13881,8 @@ define amdgpu_kernel void @global_cluster_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14196,6 +14280,8 @@ define amdgpu_kernel void @global_cluster_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14593,6 +14679,8 @@ define amdgpu_kernel void @global_cluster_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14990,6 +15078,8 @@ define amdgpu_kernel void @global_cluster_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15387,6 +15477,8 @@ define amdgpu_kernel void @global_cluster_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15642,6 +15734,8 @@ define amdgpu_kernel void @global_cluster_one_as_unordered_load(
;
; GFX1250-LABEL: global_cluster_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15884,6 +15978,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_load(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16150,6 +16246,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_load(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16440,6 +16538,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_load(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16666,6 +16766,8 @@ define amdgpu_kernel void @global_cluster_one_as_unordered_store(
;
; GFX1250-LABEL: global_cluster_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16888,6 +16990,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_store(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17127,6 +17231,8 @@ define amdgpu_kernel void @global_cluster_one_as_release_store(
;
; GFX1250-LABEL: global_cluster_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17368,6 +17474,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_store(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17592,6 +17700,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17843,6 +17953,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18092,6 +18204,8 @@ define amdgpu_kernel void @global_cluster_one_as_release_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18370,6 +18484,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18650,6 +18766,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18920,6 +19038,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19215,6 +19335,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19512,6 +19634,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19830,6 +19954,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20178,6 +20304,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20524,6 +20652,8 @@ define amdgpu_kernel void @global_cluster_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20899,6 +21029,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21276,6 +21408,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21628,6 +21762,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21978,6 +22114,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22353,6 +22491,8 @@ define amdgpu_kernel void @global_cluster_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22730,6 +22870,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23107,6 +23249,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23484,6 +23628,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23861,6 +24007,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24238,6 +24386,8 @@ define amdgpu_kernel void @global_cluster_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24615,6 +24765,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24992,6 +25144,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25345,6 +25499,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25714,6 +25870,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26109,6 +26267,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26506,6 +26666,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26878,6 +27040,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27248,6 +27412,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27643,6 +27809,8 @@ define amdgpu_kernel void @global_cluster_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28040,6 +28208,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28437,6 +28607,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28834,6 +29006,8 @@ define amdgpu_kernel void @global_cluster_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29231,6 +29405,8 @@ define amdgpu_kernel void @global_cluster_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29628,6 +29804,8 @@ define amdgpu_kernel void @global_cluster_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30025,6 +30203,8 @@ define amdgpu_kernel void @global_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30422,6 +30602,8 @@ define amdgpu_kernel void @global_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
index 6a81b0655117e..579078d6023e4 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
@@ -23,6 +23,8 @@ define amdgpu_kernel void @global_last_use_load_0(ptr addrspace(1) %in, ptr addr
;
; GFX1250-LABEL: global_last_use_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -94,6 +96,8 @@ define amdgpu_kernel void @global_last_use_load_1(ptr addrspace(1) %in, ptr addr
;
; GFX1250-LABEL: global_last_use_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -157,6 +161,8 @@ define amdgpu_kernel void @global_last_use_and_volatile_load(ptr addrspace(1) %i
;
; GFX1250-LABEL: global_last_use_and_volatile_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -232,6 +238,8 @@ define amdgpu_kernel void @global_last_use_and_nontemporal_load(ptr addrspace(1)
;
; GFX1250-LABEL: global_last_use_and_nontemporal_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index 21b5791ede4cd..134a4eaf506a3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -233,6 +233,8 @@ define amdgpu_kernel void @global_nontemporal_load_0(
;
; GFX1250-LABEL: global_nontemporal_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -659,6 +661,8 @@ define amdgpu_kernel void @global_nontemporal_load_1(
;
; GFX1250-LABEL: global_nontemporal_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -904,6 +908,8 @@ define amdgpu_kernel void @global_nontemporal_store_0(
;
; GFX1250-LABEL: global_nontemporal_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1301,6 +1307,8 @@ define amdgpu_kernel void @global_nontemporal_store_1(
;
; GFX1250-LABEL: global_nontemporal_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1551,6 +1559,8 @@ define amdgpu_kernel void @global_nontemporal_volatile_load(
;
; GFX1250-LABEL: global_nontemporal_volatile_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
index 6ada3de1aca5a..98f69f228efba 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
@@ -236,6 +236,8 @@ define amdgpu_kernel void @global_singlethread_unordered_load(
;
; GFX1250-LABEL: global_singlethread_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -478,6 +480,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_load(
;
; GFX1250-LABEL: global_singlethread_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -720,6 +724,8 @@ define amdgpu_kernel void @global_singlethread_acquire_load(
;
; GFX1250-LABEL: global_singlethread_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -962,6 +968,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_load(
;
; GFX1250-LABEL: global_singlethread_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1185,6 +1193,8 @@ define amdgpu_kernel void @global_singlethread_unordered_store(
;
; GFX1250-LABEL: global_singlethread_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1407,6 +1417,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_store(
;
; GFX1250-LABEL: global_singlethread_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1629,6 +1641,8 @@ define amdgpu_kernel void @global_singlethread_release_store(
;
; GFX1250-LABEL: global_singlethread_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1851,6 +1865,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_store(
;
; GFX1250-LABEL: global_singlethread_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2073,6 +2089,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2295,6 +2313,8 @@ define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2517,6 +2537,8 @@ define amdgpu_kernel void @global_singlethread_release_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2739,6 +2761,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2961,6 +2985,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3211,6 +3237,8 @@ define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3464,6 +3492,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3717,6 +3747,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4032,6 +4064,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4351,6 +4385,8 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4670,6 +4706,8 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4989,6 +5027,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5308,6 +5348,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5627,6 +5669,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5946,6 +5990,8 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6265,6 +6311,8 @@ define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6584,6 +6632,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6903,6 +6953,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7222,6 +7274,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7541,6 +7595,8 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7860,6 +7916,8 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8179,6 +8237,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8498,6 +8558,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8847,6 +8909,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9200,6 +9264,8 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9553,6 +9619,8 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9906,6 +9974,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10259,6 +10329,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10612,6 +10684,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10965,6 +11039,8 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11318,6 +11394,8 @@ define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11671,6 +11749,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12024,6 +12104,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12377,6 +12459,8 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12730,6 +12814,8 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13083,6 +13169,8 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13436,6 +13524,8 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13789,6 +13879,8 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14041,6 +14133,8 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
;
; GFX1250-LABEL: global_singlethread_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14283,6 +14377,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14525,6 +14621,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14767,6 +14865,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14990,6 +15090,8 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
;
; GFX1250-LABEL: global_singlethread_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15212,6 +15314,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15434,6 +15538,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_store(
;
; GFX1250-LABEL: global_singlethread_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15656,6 +15762,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15878,6 +15986,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16100,6 +16210,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16322,6 +16434,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16544,6 +16658,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16766,6 +16882,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17016,6 +17134,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17269,6 +17389,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17522,6 +17644,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17837,6 +17961,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18156,6 +18282,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18475,6 +18603,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18794,6 +18924,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19113,6 +19245,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19432,6 +19566,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19751,6 +19887,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20070,6 +20208,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20389,6 +20529,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20708,6 +20850,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21027,6 +21171,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21346,6 +21492,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21665,6 +21813,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21984,6 +22134,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22303,6 +22455,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22652,6 +22806,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cm
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23005,6 +23161,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpx
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23358,6 +23516,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpx
;
; GFX1250-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23711,6 +23871,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpx
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24064,6 +24226,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpx
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24417,6 +24581,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpx
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24770,6 +24936,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25123,6 +25291,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25476,6 +25646,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25829,6 +26001,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26182,6 +26356,8 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpx
;
; GFX1250-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26535,6 +26711,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26888,6 +27066,8 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27241,6 +27421,8 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27594,6 +27776,8 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxch
;
; GFX1250-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
index ceff9d4bf08f9..e7b0aa691b4a9 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
@@ -236,6 +236,8 @@ define amdgpu_kernel void @global_system_unordered_load(
;
; GFX1250-LABEL: global_system_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -478,6 +480,8 @@ define amdgpu_kernel void @global_system_monotonic_load(
;
; GFX1250-LABEL: global_system_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -746,6 +750,8 @@ define amdgpu_kernel void @global_system_acquire_load(
;
; GFX1250-LABEL: global_system_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1030,6 +1036,8 @@ define amdgpu_kernel void @global_system_seq_cst_load(
;
; GFX1250-LABEL: global_system_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1257,6 +1265,8 @@ define amdgpu_kernel void @global_system_unordered_store(
;
; GFX1250-LABEL: global_system_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1479,6 +1489,8 @@ define amdgpu_kernel void @global_system_monotonic_store(
;
; GFX1250-LABEL: global_system_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1722,6 +1734,8 @@ define amdgpu_kernel void @global_system_release_store(
;
; GFX1250-LABEL: global_system_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1970,6 +1984,8 @@ define amdgpu_kernel void @global_system_seq_cst_store(
;
; GFX1250-LABEL: global_system_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2197,6 +2213,8 @@ define amdgpu_kernel void @global_system_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_system_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2450,6 +2468,8 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw(
;
; GFX1250-LABEL: global_system_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2704,6 +2724,8 @@ define amdgpu_kernel void @global_system_release_atomicrmw(
;
; GFX1250-LABEL: global_system_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2991,6 +3013,8 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_system_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3281,6 +3305,8 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_system_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3557,6 +3583,8 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_system_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3859,6 +3887,8 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_system_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4166,6 +4196,8 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_system_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4488,6 +4520,8 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4838,6 +4872,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5189,6 +5225,8 @@ define amdgpu_kernel void @global_system_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5573,6 +5611,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5960,6 +6000,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6318,6 +6360,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6671,6 +6715,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7053,6 +7099,8 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7440,6 +7488,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7827,6 +7877,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8214,6 +8266,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8571,6 +8625,8 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8942,6 +8998,8 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9344,6 +9402,8 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9751,6 +9811,8 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10129,6 +10191,8 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10502,6 +10566,8 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10904,6 +10970,8 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11311,6 +11379,8 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11718,6 +11788,8 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12125,6 +12197,8 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12532,6 +12606,8 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12939,6 +13015,8 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13346,6 +13424,8 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13753,6 +13833,8 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14012,6 +14094,8 @@ define amdgpu_kernel void @global_system_one_as_unordered_load(
;
; GFX1250-LABEL: global_system_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14254,6 +14338,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_load(
;
; GFX1250-LABEL: global_system_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14522,6 +14608,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_load(
;
; GFX1250-LABEL: global_system_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14815,6 +14903,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15042,6 +15132,8 @@ define amdgpu_kernel void @global_system_one_as_unordered_store(
;
; GFX1250-LABEL: global_system_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15264,6 +15356,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_store(
;
; GFX1250-LABEL: global_system_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15507,6 +15601,8 @@ define amdgpu_kernel void @global_system_one_as_release_store(
;
; GFX1250-LABEL: global_system_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15755,6 +15851,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_store(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15982,6 +16080,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16235,6 +16335,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16489,6 +16591,8 @@ define amdgpu_kernel void @global_system_one_as_release_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16776,6 +16880,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17066,6 +17172,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17342,6 +17450,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17644,6 +17754,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17951,6 +18063,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18273,6 +18387,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18623,6 +18739,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18974,6 +19092,8 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19358,6 +19478,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19745,6 +19867,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20103,6 +20227,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20456,6 +20582,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20838,6 +20966,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21225,6 +21355,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21612,6 +21744,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21999,6 +22133,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22386,6 +22522,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22773,6 +22911,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23160,6 +23300,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23547,6 +23689,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23904,6 +24048,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24275,6 +24421,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24659,6 +24807,8 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25064,6 +25214,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25471,6 +25623,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25849,6 +26003,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26222,6 +26378,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26624,6 +26782,8 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27031,6 +27191,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27438,6 +27600,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27845,6 +28009,8 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28252,6 +28418,8 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28659,6 +28827,8 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29066,6 +29236,8 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29473,6 +29645,8 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
index ac60870b362b8..f96ec16cc6279 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
@@ -179,6 +179,8 @@ define amdgpu_kernel void @global_volatile_load_0(
;
; GFX1250-LABEL: global_volatile_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -494,6 +496,8 @@ define amdgpu_kernel void @global_volatile_load_1(
;
; GFX1250-LABEL: global_volatile_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
@@ -699,6 +703,8 @@ define amdgpu_kernel void @global_volatile_store_0(
;
; GFX1250-LABEL: global_volatile_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1007,6 +1013,8 @@ define amdgpu_kernel void @global_volatile_store_1(
;
; GFX1250-LABEL: global_volatile_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1205,6 +1213,8 @@ define amdgpu_kernel void @global_volatile_workgroup_acquire_load(
;
; GFX1250-LABEL: global_volatile_workgroup_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1387,6 +1397,8 @@ define amdgpu_kernel void @global_volatile_workgroup_release_store(
;
; GFX1250-LABEL: global_volatile_workgroup_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
index 04c8bf154d7ad..fb965fe11698e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
@@ -236,6 +236,8 @@ define amdgpu_kernel void @global_wavefront_unordered_load(
;
; GFX1250-LABEL: global_wavefront_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -478,6 +480,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_load(
;
; GFX1250-LABEL: global_wavefront_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -720,6 +724,8 @@ define amdgpu_kernel void @global_wavefront_acquire_load(
;
; GFX1250-LABEL: global_wavefront_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -962,6 +968,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_load(
;
; GFX1250-LABEL: global_wavefront_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1185,6 +1193,8 @@ define amdgpu_kernel void @global_wavefront_unordered_store(
;
; GFX1250-LABEL: global_wavefront_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1407,6 +1417,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_store(
;
; GFX1250-LABEL: global_wavefront_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1629,6 +1641,8 @@ define amdgpu_kernel void @global_wavefront_release_store(
;
; GFX1250-LABEL: global_wavefront_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1851,6 +1865,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_store(
;
; GFX1250-LABEL: global_wavefront_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2073,6 +2089,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2295,6 +2313,8 @@ define amdgpu_kernel void @global_wavefront_acquire_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2517,6 +2537,8 @@ define amdgpu_kernel void @global_wavefront_release_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2739,6 +2761,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2961,6 +2985,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3211,6 +3237,8 @@ define amdgpu_kernel void @global_wavefront_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3464,6 +3492,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3717,6 +3747,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4032,6 +4064,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4351,6 +4385,8 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4670,6 +4706,8 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4989,6 +5027,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5308,6 +5348,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5627,6 +5669,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5946,6 +5990,8 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6265,6 +6311,8 @@ define amdgpu_kernel void @global_wavefront_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6584,6 +6632,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6903,6 +6953,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7222,6 +7274,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7541,6 +7595,8 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7860,6 +7916,8 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8179,6 +8237,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8498,6 +8558,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8847,6 +8909,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9200,6 +9264,8 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9553,6 +9619,8 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9906,6 +9974,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10259,6 +10329,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10612,6 +10684,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10965,6 +11039,8 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11318,6 +11394,8 @@ define amdgpu_kernel void @global_wavefront_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11671,6 +11749,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12024,6 +12104,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12377,6 +12459,8 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12730,6 +12814,8 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13083,6 +13169,8 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13436,6 +13524,8 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13789,6 +13879,8 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14041,6 +14133,8 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_load(
;
; GFX1250-LABEL: global_wavefront_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14283,6 +14377,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_load(
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14525,6 +14621,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_load(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14767,6 +14865,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_load(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14990,6 +15090,8 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_store(
;
; GFX1250-LABEL: global_wavefront_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15212,6 +15314,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_store(
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15434,6 +15538,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_store(
;
; GFX1250-LABEL: global_wavefront_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15656,6 +15762,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_store(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15878,6 +15986,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16100,6 +16210,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16322,6 +16434,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16544,6 +16658,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16766,6 +16882,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17016,6 +17134,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17269,6 +17389,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17522,6 +17644,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17837,6 +17961,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18156,6 +18282,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18475,6 +18603,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18794,6 +18924,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19113,6 +19245,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19432,6 +19566,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19751,6 +19887,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20070,6 +20208,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20389,6 +20529,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20708,6 +20850,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21027,6 +21171,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21346,6 +21492,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21665,6 +21813,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21984,6 +22134,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22303,6 +22455,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22652,6 +22806,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_ret_cmpxc
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23005,6 +23161,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23358,6 +23516,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_wavefront_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23711,6 +23871,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24064,6 +24226,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24417,6 +24581,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_ret_cmpxchg
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24770,6 +24936,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25123,6 +25291,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25476,6 +25646,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25829,6 +26001,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26182,6 +26356,8 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg
;
; GFX1250-LABEL: global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26535,6 +26711,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26888,6 +27066,8 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27241,6 +27421,8 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27594,6 +27776,8 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
index ac5a65c5abdd3..9be0dbe1d0605 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
@@ -236,6 +236,8 @@ define amdgpu_kernel void @global_workgroup_unordered_load(
;
; GFX1250-LABEL: global_workgroup_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -478,6 +480,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_load(
;
; GFX1250-LABEL: global_workgroup_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -729,6 +733,8 @@ define amdgpu_kernel void @global_workgroup_acquire_load(
;
; GFX1250-LABEL: global_workgroup_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -998,6 +1004,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_load(
;
; GFX1250-LABEL: global_workgroup_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1223,6 +1231,8 @@ define amdgpu_kernel void @global_workgroup_unordered_store(
;
; GFX1250-LABEL: global_workgroup_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1445,6 +1455,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_store(
;
; GFX1250-LABEL: global_workgroup_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1682,6 +1694,8 @@ define amdgpu_kernel void @global_workgroup_release_store(
;
; GFX1250-LABEL: global_workgroup_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1921,6 +1935,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_store(
;
; GFX1250-LABEL: global_workgroup_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2145,6 +2161,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2377,6 +2395,8 @@ define amdgpu_kernel void @global_workgroup_acquire_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2623,6 +2643,8 @@ define amdgpu_kernel void @global_workgroup_release_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2883,6 +2905,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3144,6 +3168,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3402,6 +3428,8 @@ define amdgpu_kernel void @global_workgroup_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3683,6 +3711,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3966,6 +3996,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4283,6 +4315,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4612,6 +4646,8 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4955,6 +4991,8 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5312,6 +5350,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5670,6 +5710,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6002,6 +6044,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6332,6 +6376,8 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6688,6 +6734,8 @@ define amdgpu_kernel void @global_workgroup_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7046,6 +7094,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7404,6 +7454,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7759,6 +7811,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8114,6 +8168,8 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8472,6 +8528,8 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8830,6 +8888,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9188,6 +9248,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9540,6 +9602,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9898,6 +9962,8 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10274,6 +10340,8 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10657,6 +10725,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11040,6 +11110,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11400,6 +11472,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11758,6 +11832,8 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12139,6 +12215,8 @@ define amdgpu_kernel void @global_workgroup_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12522,6 +12600,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12905,6 +12985,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13288,6 +13370,8 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13671,6 +13755,8 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14054,6 +14140,8 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14437,6 +14525,8 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14820,6 +14910,8 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15074,6 +15166,8 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_load(
;
; GFX1250-LABEL: global_workgroup_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15316,6 +15410,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_load(
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15567,6 +15663,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_load(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15836,6 +15934,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_load(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16061,6 +16161,8 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_store(
;
; GFX1250-LABEL: global_workgroup_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16283,6 +16385,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_store(
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16517,6 +16621,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_store(
;
; GFX1250-LABEL: global_workgroup_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16753,6 +16859,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_store(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16977,6 +17085,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17209,6 +17319,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17450,6 +17562,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17705,6 +17819,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17961,6 +18077,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18219,6 +18337,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18495,6 +18615,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18773,6 +18895,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19090,6 +19214,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19419,6 +19545,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19757,6 +19885,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20109,6 +20239,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20462,6 +20594,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20794,6 +20928,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21124,6 +21260,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21475,6 +21613,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21828,6 +21968,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22181,6 +22323,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22531,6 +22675,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22881,6 +23027,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23234,6 +23382,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23587,6 +23737,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23940,6 +24092,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24292,6 +24446,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_ret_cmpxc
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24650,6 +24806,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25021,6 +25179,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25399,6 +25559,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25777,6 +25939,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26137,6 +26301,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_ret_cmpxchg
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26495,6 +26661,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26871,6 +27039,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27249,6 +27419,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27627,6 +27799,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28005,6 +28179,8 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg
;
; GFX1250-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28383,6 +28559,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28761,6 +28939,8 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29139,6 +29319,8 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29517,6 +29699,8 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
index 75d19b4ca86c8..22705d588a391 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
@@ -217,6 +217,8 @@ define amdgpu_kernel void @local_agent_unordered_load(
;
; GFX1250-LABEL: local_agent_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -441,6 +443,8 @@ define amdgpu_kernel void @local_agent_monotonic_load(
;
; GFX1250-LABEL: local_agent_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -670,6 +674,8 @@ define amdgpu_kernel void @local_agent_acquire_load(
;
; GFX1250-LABEL: local_agent_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -922,6 +928,8 @@ define amdgpu_kernel void @local_agent_seq_cst_load(
;
; GFX1250-LABEL: local_agent_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1131,6 +1139,8 @@ define amdgpu_kernel void @local_agent_unordered_store(
;
; GFX1250-LABEL: local_agent_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1335,6 +1345,8 @@ define amdgpu_kernel void @local_agent_monotonic_store(
;
; GFX1250-LABEL: local_agent_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1562,6 +1574,8 @@ define amdgpu_kernel void @local_agent_release_store(
;
; GFX1250-LABEL: local_agent_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1791,6 +1805,8 @@ define amdgpu_kernel void @local_agent_seq_cst_store(
;
; GFX1250-LABEL: local_agent_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1997,6 +2013,8 @@ define amdgpu_kernel void @local_agent_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_agent_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2217,6 +2235,8 @@ define amdgpu_kernel void @local_agent_acquire_atomicrmw(
;
; GFX1250-LABEL: local_agent_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2445,6 +2465,8 @@ define amdgpu_kernel void @local_agent_release_atomicrmw(
;
; GFX1250-LABEL: local_agent_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2690,6 +2712,8 @@ define amdgpu_kernel void @local_agent_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_agent_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2936,6 +2960,8 @@ define amdgpu_kernel void @local_agent_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_agent_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3190,6 +3216,8 @@ define amdgpu_kernel void @local_agent_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_agent_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3468,6 +3496,8 @@ define amdgpu_kernel void @local_agent_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_agent_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3748,6 +3778,8 @@ define amdgpu_kernel void @local_agent_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_agent_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4010,6 +4042,8 @@ define amdgpu_kernel void @local_agent_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4287,6 +4321,8 @@ define amdgpu_kernel void @local_agent_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4572,6 +4608,8 @@ define amdgpu_kernel void @local_agent_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4874,6 +4912,8 @@ define amdgpu_kernel void @local_agent_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5177,6 +5217,8 @@ define amdgpu_kernel void @local_agent_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5457,6 +5499,8 @@ define amdgpu_kernel void @local_agent_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5735,6 +5779,8 @@ define amdgpu_kernel void @local_agent_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6036,6 +6082,8 @@ define amdgpu_kernel void @local_agent_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6339,6 +6387,8 @@ define amdgpu_kernel void @local_agent_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6642,6 +6692,8 @@ define amdgpu_kernel void @local_agent_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6945,6 +6997,8 @@ define amdgpu_kernel void @local_agent_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7248,6 +7302,8 @@ define amdgpu_kernel void @local_agent_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7551,6 +7607,8 @@ define amdgpu_kernel void @local_agent_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7854,6 +7912,8 @@ define amdgpu_kernel void @local_agent_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8157,6 +8217,8 @@ define amdgpu_kernel void @local_agent_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8463,6 +8525,8 @@ define amdgpu_kernel void @local_agent_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8776,6 +8840,8 @@ define amdgpu_kernel void @local_agent_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9107,6 +9173,8 @@ define amdgpu_kernel void @local_agent_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9445,6 +9513,8 @@ define amdgpu_kernel void @local_agent_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9783,6 +9853,8 @@ define amdgpu_kernel void @local_agent_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10098,6 +10170,8 @@ define amdgpu_kernel void @local_agent_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10411,6 +10485,8 @@ define amdgpu_kernel void @local_agent_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10747,6 +10823,8 @@ define amdgpu_kernel void @local_agent_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11085,6 +11163,8 @@ define amdgpu_kernel void @local_agent_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11423,6 +11503,8 @@ define amdgpu_kernel void @local_agent_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11761,6 +11843,8 @@ define amdgpu_kernel void @local_agent_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12099,6 +12183,8 @@ define amdgpu_kernel void @local_agent_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12437,6 +12523,8 @@ define amdgpu_kernel void @local_agent_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12775,6 +12863,8 @@ define amdgpu_kernel void @local_agent_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13113,6 +13203,8 @@ define amdgpu_kernel void @local_agent_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13346,6 +13438,8 @@ define amdgpu_kernel void @local_agent_one_as_unordered_load(
;
; GFX1250-LABEL: local_agent_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13570,6 +13664,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_load(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13794,6 +13890,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_load(
;
; GFX1250-LABEL: local_agent_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14018,6 +14116,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_load(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14225,6 +14325,8 @@ define amdgpu_kernel void @local_agent_one_as_unordered_store(
;
; GFX1250-LABEL: local_agent_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14429,6 +14531,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_store(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14633,6 +14737,8 @@ define amdgpu_kernel void @local_agent_one_as_release_store(
;
; GFX1250-LABEL: local_agent_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14837,6 +14943,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_store(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15041,6 +15149,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15245,6 +15355,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15449,6 +15561,8 @@ define amdgpu_kernel void @local_agent_one_as_release_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15653,6 +15767,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15857,6 +15973,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16103,6 +16221,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16353,6 +16473,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16603,6 +16725,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16863,6 +16987,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17124,6 +17250,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17385,6 +17513,8 @@ define amdgpu_kernel void @local_agent_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17646,6 +17776,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17907,6 +18039,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18168,6 +18302,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18429,6 +18565,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18690,6 +18828,8 @@ define amdgpu_kernel void @local_agent_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18951,6 +19091,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19212,6 +19354,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19473,6 +19617,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19734,6 +19880,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19995,6 +20143,8 @@ define amdgpu_kernel void @local_agent_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20256,6 +20406,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20517,6 +20669,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20820,6 +20974,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21128,6 +21284,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21436,6 +21594,8 @@ define amdgpu_kernel void @local_agent_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21744,6 +21904,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22052,6 +22214,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22360,6 +22524,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22668,6 +22834,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22976,6 +23144,8 @@ define amdgpu_kernel void @local_agent_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23284,6 +23454,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23592,6 +23764,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23900,6 +24074,8 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24208,6 +24384,8 @@ define amdgpu_kernel void @local_agent_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24516,6 +24694,8 @@ define amdgpu_kernel void @local_agent_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24824,6 +25004,8 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25132,6 +25314,8 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-cluster.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-cluster.ll
index 6a48822ec1d51..0c5f03e47906d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-cluster.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-cluster.ll
@@ -217,6 +217,8 @@ define amdgpu_kernel void @local_cluster_unordered_load(
;
; GFX1250-LABEL: local_cluster_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -441,6 +443,8 @@ define amdgpu_kernel void @local_cluster_monotonic_load(
;
; GFX1250-LABEL: local_cluster_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -670,6 +674,8 @@ define amdgpu_kernel void @local_cluster_acquire_load(
;
; GFX1250-LABEL: local_cluster_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -922,6 +928,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_load(
;
; GFX1250-LABEL: local_cluster_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1131,6 +1139,8 @@ define amdgpu_kernel void @local_cluster_unordered_store(
;
; GFX1250-LABEL: local_cluster_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1335,6 +1345,8 @@ define amdgpu_kernel void @local_cluster_monotonic_store(
;
; GFX1250-LABEL: local_cluster_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1562,6 +1574,8 @@ define amdgpu_kernel void @local_cluster_release_store(
;
; GFX1250-LABEL: local_cluster_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1791,6 +1805,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_store(
;
; GFX1250-LABEL: local_cluster_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1997,6 +2013,8 @@ define amdgpu_kernel void @local_cluster_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_cluster_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2217,6 +2235,8 @@ define amdgpu_kernel void @local_cluster_acquire_atomicrmw(
;
; GFX1250-LABEL: local_cluster_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2445,6 +2465,8 @@ define amdgpu_kernel void @local_cluster_release_atomicrmw(
;
; GFX1250-LABEL: local_cluster_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2690,6 +2712,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_cluster_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2936,6 +2960,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_cluster_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3190,6 +3216,8 @@ define amdgpu_kernel void @local_cluster_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_cluster_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3468,6 +3496,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_cluster_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3748,6 +3778,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_cluster_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4010,6 +4042,8 @@ define amdgpu_kernel void @local_cluster_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4287,6 +4321,8 @@ define amdgpu_kernel void @local_cluster_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4572,6 +4608,8 @@ define amdgpu_kernel void @local_cluster_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4874,6 +4912,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5177,6 +5217,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5457,6 +5499,8 @@ define amdgpu_kernel void @local_cluster_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5735,6 +5779,8 @@ define amdgpu_kernel void @local_cluster_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6036,6 +6082,8 @@ define amdgpu_kernel void @local_cluster_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6339,6 +6387,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6642,6 +6692,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6945,6 +6997,8 @@ define amdgpu_kernel void @local_cluster_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7248,6 +7302,8 @@ define amdgpu_kernel void @local_cluster_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7551,6 +7607,8 @@ define amdgpu_kernel void @local_cluster_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7854,6 +7912,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8157,6 +8217,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8463,6 +8525,8 @@ define amdgpu_kernel void @local_cluster_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8776,6 +8840,8 @@ define amdgpu_kernel void @local_cluster_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9107,6 +9173,8 @@ define amdgpu_kernel void @local_cluster_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9445,6 +9513,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9783,6 +9853,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10098,6 +10170,8 @@ define amdgpu_kernel void @local_cluster_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10411,6 +10485,8 @@ define amdgpu_kernel void @local_cluster_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10747,6 +10823,8 @@ define amdgpu_kernel void @local_cluster_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11085,6 +11163,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11423,6 +11503,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11761,6 +11843,8 @@ define amdgpu_kernel void @local_cluster_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12099,6 +12183,8 @@ define amdgpu_kernel void @local_cluster_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12437,6 +12523,8 @@ define amdgpu_kernel void @local_cluster_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12775,6 +12863,8 @@ define amdgpu_kernel void @local_cluster_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13113,6 +13203,8 @@ define amdgpu_kernel void @local_cluster_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13346,6 +13438,8 @@ define amdgpu_kernel void @local_cluster_one_as_unordered_load(
;
; GFX1250-LABEL: local_cluster_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13570,6 +13664,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_load(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13794,6 +13890,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_load(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14018,6 +14116,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_load(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14225,6 +14325,8 @@ define amdgpu_kernel void @local_cluster_one_as_unordered_store(
;
; GFX1250-LABEL: local_cluster_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14429,6 +14531,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_store(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14633,6 +14737,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_store(
;
; GFX1250-LABEL: local_cluster_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14837,6 +14943,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_store(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15041,6 +15149,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15245,6 +15355,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15449,6 +15561,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15653,6 +15767,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15857,6 +15973,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16103,6 +16221,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16353,6 +16473,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16603,6 +16725,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16863,6 +16987,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17124,6 +17250,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17385,6 +17513,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17646,6 +17776,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17907,6 +18039,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18168,6 +18302,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18429,6 +18565,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18690,6 +18828,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18951,6 +19091,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19212,6 +19354,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19473,6 +19617,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19734,6 +19880,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19995,6 +20143,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20256,6 +20406,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20517,6 +20669,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20820,6 +20974,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21128,6 +21284,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21436,6 +21594,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21744,6 +21904,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22052,6 +22214,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22360,6 +22524,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22668,6 +22834,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22976,6 +23144,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23284,6 +23454,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23592,6 +23764,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23900,6 +24074,8 @@ define amdgpu_kernel void @local_cluster_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24208,6 +24384,8 @@ define amdgpu_kernel void @local_cluster_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24516,6 +24694,8 @@ define amdgpu_kernel void @local_cluster_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24824,6 +25004,8 @@ define amdgpu_kernel void @local_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25132,6 +25314,8 @@ define amdgpu_kernel void @local_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 55a315787742c..025aff3df9ca2 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -235,6 +235,8 @@ define amdgpu_kernel void @local_nontemporal_load_0(
;
; GFX1250-LABEL: local_nontemporal_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -525,6 +527,8 @@ define amdgpu_kernel void @local_nontemporal_load_1(
;
; GFX1250-LABEL: local_nontemporal_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
@@ -752,6 +756,8 @@ define amdgpu_kernel void @local_nontemporal_store_0(
;
; GFX1250-LABEL: local_nontemporal_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1020,6 +1026,8 @@ define amdgpu_kernel void @local_nontemporal_store_1(
;
; GFX1250-LABEL: local_nontemporal_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1268,6 +1276,8 @@ define amdgpu_kernel void @local_nontemporal_volatile_load(
;
; GFX1250-LABEL: local_nontemporal_volatile_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
index de54a612bbcff..b9255fd96b673 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
@@ -217,6 +217,8 @@ define amdgpu_kernel void @local_singlethread_unordered_load(
;
; GFX1250-LABEL: local_singlethread_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -441,6 +443,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_load(
;
; GFX1250-LABEL: local_singlethread_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -665,6 +669,8 @@ define amdgpu_kernel void @local_singlethread_acquire_load(
;
; GFX1250-LABEL: local_singlethread_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -889,6 +895,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_load(
;
; GFX1250-LABEL: local_singlethread_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1096,6 +1104,8 @@ define amdgpu_kernel void @local_singlethread_unordered_store(
;
; GFX1250-LABEL: local_singlethread_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1300,6 +1310,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_store(
;
; GFX1250-LABEL: local_singlethread_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1504,6 +1516,8 @@ define amdgpu_kernel void @local_singlethread_release_store(
;
; GFX1250-LABEL: local_singlethread_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1708,6 +1722,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_store(
;
; GFX1250-LABEL: local_singlethread_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1912,6 +1928,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2116,6 +2134,8 @@ define amdgpu_kernel void @local_singlethread_acquire_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2320,6 +2340,8 @@ define amdgpu_kernel void @local_singlethread_release_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2524,6 +2546,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2728,6 +2752,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2974,6 +3000,8 @@ define amdgpu_kernel void @local_singlethread_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3224,6 +3252,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3474,6 +3504,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3734,6 +3766,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3995,6 +4029,8 @@ define amdgpu_kernel void @local_singlethread_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4256,6 +4292,8 @@ define amdgpu_kernel void @local_singlethread_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4517,6 +4555,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4778,6 +4818,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5039,6 +5081,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5300,6 +5344,8 @@ define amdgpu_kernel void @local_singlethread_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5561,6 +5607,8 @@ define amdgpu_kernel void @local_singlethread_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5822,6 +5870,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6083,6 +6133,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6344,6 +6396,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6605,6 +6659,8 @@ define amdgpu_kernel void @local_singlethread_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6866,6 +6922,8 @@ define amdgpu_kernel void @local_singlethread_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7127,6 +7185,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7388,6 +7448,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7691,6 +7753,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7999,6 +8063,8 @@ define amdgpu_kernel void @local_singlethread_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8307,6 +8373,8 @@ define amdgpu_kernel void @local_singlethread_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8615,6 +8683,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8923,6 +8993,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9231,6 +9303,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9539,6 +9613,8 @@ define amdgpu_kernel void @local_singlethread_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9847,6 +9923,8 @@ define amdgpu_kernel void @local_singlethread_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10155,6 +10233,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10463,6 +10543,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10771,6 +10853,8 @@ define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11079,6 +11163,8 @@ define amdgpu_kernel void @local_singlethread_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11387,6 +11473,8 @@ define amdgpu_kernel void @local_singlethread_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11695,6 +11783,8 @@ define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12003,6 +12093,8 @@ define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12234,6 +12326,8 @@ define amdgpu_kernel void @local_singlethread_one_as_unordered_load(
;
; GFX1250-LABEL: local_singlethread_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12458,6 +12552,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_load(
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12682,6 +12778,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_load(
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12906,6 +13004,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_load(
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13113,6 +13213,8 @@ define amdgpu_kernel void @local_singlethread_one_as_unordered_store(
;
; GFX1250-LABEL: local_singlethread_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13317,6 +13419,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_store(
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13521,6 +13625,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_store(
;
; GFX1250-LABEL: local_singlethread_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13725,6 +13831,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_store(
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13929,6 +14037,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14133,6 +14243,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14337,6 +14449,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14541,6 +14655,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14745,6 +14861,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14991,6 +15109,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15241,6 +15361,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15491,6 +15613,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15751,6 +15875,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16012,6 +16138,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16273,6 +16401,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16534,6 +16664,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16795,6 +16927,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17056,6 +17190,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17317,6 +17453,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17578,6 +17716,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17839,6 +17979,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18100,6 +18242,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18361,6 +18505,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18622,6 +18768,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18883,6 +19031,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19144,6 +19294,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19405,6 +19557,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19708,6 +19862,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_ret_cmp
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20016,6 +20172,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_ret_cmpxc
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20324,6 +20482,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_ret_cmpxc
;
; GFX1250-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20632,6 +20792,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_ret_cmpxc
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20940,6 +21102,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_ret_cmpxc
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21248,6 +21412,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_ret_cmpxc
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21556,6 +21722,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21864,6 +22032,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_acquire_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22172,6 +22342,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22480,6 +22652,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22788,6 +22962,8 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_ret_cmpxc
;
; GFX1250-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23096,6 +23272,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23404,6 +23582,8 @@ define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23712,6 +23892,8 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24020,6 +24202,8 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg
;
; GFX1250-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
index 6ec04cf368ffb..4fe26ff7a2dd1 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
@@ -217,6 +217,8 @@ define amdgpu_kernel void @local_system_unordered_load(
;
; GFX1250-LABEL: local_system_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -441,6 +443,8 @@ define amdgpu_kernel void @local_system_monotonic_load(
;
; GFX1250-LABEL: local_system_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -670,6 +674,8 @@ define amdgpu_kernel void @local_system_acquire_load(
;
; GFX1250-LABEL: local_system_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -922,6 +928,8 @@ define amdgpu_kernel void @local_system_seq_cst_load(
;
; GFX1250-LABEL: local_system_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1131,6 +1139,8 @@ define amdgpu_kernel void @local_system_unordered_store(
;
; GFX1250-LABEL: local_system_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1335,6 +1345,8 @@ define amdgpu_kernel void @local_system_monotonic_store(
;
; GFX1250-LABEL: local_system_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1562,6 +1574,8 @@ define amdgpu_kernel void @local_system_release_store(
;
; GFX1250-LABEL: local_system_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1791,6 +1805,8 @@ define amdgpu_kernel void @local_system_seq_cst_store(
;
; GFX1250-LABEL: local_system_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1997,6 +2013,8 @@ define amdgpu_kernel void @local_system_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_system_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2217,6 +2235,8 @@ define amdgpu_kernel void @local_system_acquire_atomicrmw(
;
; GFX1250-LABEL: local_system_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2445,6 +2465,8 @@ define amdgpu_kernel void @local_system_release_atomicrmw(
;
; GFX1250-LABEL: local_system_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2690,6 +2712,8 @@ define amdgpu_kernel void @local_system_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_system_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2936,6 +2960,8 @@ define amdgpu_kernel void @local_system_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_system_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3190,6 +3216,8 @@ define amdgpu_kernel void @local_system_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_system_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3468,6 +3496,8 @@ define amdgpu_kernel void @local_system_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_system_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3748,6 +3778,8 @@ define amdgpu_kernel void @local_system_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_system_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4010,6 +4042,8 @@ define amdgpu_kernel void @local_system_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4287,6 +4321,8 @@ define amdgpu_kernel void @local_system_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4572,6 +4608,8 @@ define amdgpu_kernel void @local_system_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4874,6 +4912,8 @@ define amdgpu_kernel void @local_system_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5177,6 +5217,8 @@ define amdgpu_kernel void @local_system_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5457,6 +5499,8 @@ define amdgpu_kernel void @local_system_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5735,6 +5779,8 @@ define amdgpu_kernel void @local_system_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6036,6 +6082,8 @@ define amdgpu_kernel void @local_system_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6339,6 +6387,8 @@ define amdgpu_kernel void @local_system_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6642,6 +6692,8 @@ define amdgpu_kernel void @local_system_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6945,6 +6997,8 @@ define amdgpu_kernel void @local_system_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7248,6 +7302,8 @@ define amdgpu_kernel void @local_system_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7551,6 +7607,8 @@ define amdgpu_kernel void @local_system_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7854,6 +7912,8 @@ define amdgpu_kernel void @local_system_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8157,6 +8217,8 @@ define amdgpu_kernel void @local_system_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8463,6 +8525,8 @@ define amdgpu_kernel void @local_system_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8776,6 +8840,8 @@ define amdgpu_kernel void @local_system_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9107,6 +9173,8 @@ define amdgpu_kernel void @local_system_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9445,6 +9513,8 @@ define amdgpu_kernel void @local_system_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9783,6 +9853,8 @@ define amdgpu_kernel void @local_system_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10098,6 +10170,8 @@ define amdgpu_kernel void @local_system_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10411,6 +10485,8 @@ define amdgpu_kernel void @local_system_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10747,6 +10823,8 @@ define amdgpu_kernel void @local_system_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11085,6 +11163,8 @@ define amdgpu_kernel void @local_system_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11423,6 +11503,8 @@ define amdgpu_kernel void @local_system_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11761,6 +11843,8 @@ define amdgpu_kernel void @local_system_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12099,6 +12183,8 @@ define amdgpu_kernel void @local_system_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12437,6 +12523,8 @@ define amdgpu_kernel void @local_system_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12775,6 +12863,8 @@ define amdgpu_kernel void @local_system_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13113,6 +13203,8 @@ define amdgpu_kernel void @local_system_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13346,6 +13438,8 @@ define amdgpu_kernel void @local_system_one_as_unordered_load(
;
; GFX1250-LABEL: local_system_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13570,6 +13664,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_load(
;
; GFX1250-LABEL: local_system_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13794,6 +13890,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_load(
;
; GFX1250-LABEL: local_system_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14018,6 +14116,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_load(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14225,6 +14325,8 @@ define amdgpu_kernel void @local_system_one_as_unordered_store(
;
; GFX1250-LABEL: local_system_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14429,6 +14531,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_store(
;
; GFX1250-LABEL: local_system_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14633,6 +14737,8 @@ define amdgpu_kernel void @local_system_one_as_release_store(
;
; GFX1250-LABEL: local_system_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14837,6 +14943,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_store(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15041,6 +15149,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15245,6 +15355,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15449,6 +15561,8 @@ define amdgpu_kernel void @local_system_one_as_release_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15653,6 +15767,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15857,6 +15973,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16103,6 +16221,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16353,6 +16473,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16603,6 +16725,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16863,6 +16987,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17124,6 +17250,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17385,6 +17513,8 @@ define amdgpu_kernel void @local_system_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17646,6 +17776,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17907,6 +18039,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18168,6 +18302,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18429,6 +18565,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18690,6 +18828,8 @@ define amdgpu_kernel void @local_system_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18951,6 +19091,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19212,6 +19354,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19473,6 +19617,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19734,6 +19880,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19995,6 +20143,8 @@ define amdgpu_kernel void @local_system_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20256,6 +20406,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20517,6 +20669,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20820,6 +20974,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21128,6 +21284,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21436,6 +21594,8 @@ define amdgpu_kernel void @local_system_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21744,6 +21904,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22052,6 +22214,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22360,6 +22524,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22668,6 +22834,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22976,6 +23144,8 @@ define amdgpu_kernel void @local_system_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23284,6 +23454,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23592,6 +23764,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23900,6 +24074,8 @@ define amdgpu_kernel void @local_system_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24208,6 +24384,8 @@ define amdgpu_kernel void @local_system_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24516,6 +24694,8 @@ define amdgpu_kernel void @local_system_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24824,6 +25004,8 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25132,6 +25314,8 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
index 2a16255155e18..0d80abaa3d7dd 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
@@ -171,6 +171,8 @@ define amdgpu_kernel void @local_volatile_load_0(
;
; GFX1250-LABEL: local_volatile_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -381,6 +383,8 @@ define amdgpu_kernel void @local_volatile_load_1(
;
; GFX1250-LABEL: local_volatile_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
@@ -548,6 +552,8 @@ define amdgpu_kernel void @local_volatile_store_0(
;
; GFX1250-LABEL: local_volatile_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -736,6 +742,8 @@ define amdgpu_kernel void @local_volatile_store_1(
;
; GFX1250-LABEL: local_volatile_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -909,6 +917,8 @@ define amdgpu_kernel void @local_volatile_workgroup_acquire_load(
;
; GFX1250-LABEL: local_volatile_workgroup_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1079,6 +1089,8 @@ define amdgpu_kernel void @local_volatile_workgroup_release_store(
;
; GFX1250-LABEL: local_volatile_workgroup_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
index 53074b5d06213..3b79baad07033 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
@@ -217,6 +217,8 @@ define amdgpu_kernel void @local_wavefront_unordered_load(
;
; GFX1250-LABEL: local_wavefront_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -441,6 +443,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_load(
;
; GFX1250-LABEL: local_wavefront_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -665,6 +669,8 @@ define amdgpu_kernel void @local_wavefront_acquire_load(
;
; GFX1250-LABEL: local_wavefront_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -889,6 +895,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_load(
;
; GFX1250-LABEL: local_wavefront_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1096,6 +1104,8 @@ define amdgpu_kernel void @local_wavefront_unordered_store(
;
; GFX1250-LABEL: local_wavefront_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1300,6 +1310,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_store(
;
; GFX1250-LABEL: local_wavefront_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1504,6 +1516,8 @@ define amdgpu_kernel void @local_wavefront_release_store(
;
; GFX1250-LABEL: local_wavefront_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1708,6 +1722,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_store(
;
; GFX1250-LABEL: local_wavefront_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1912,6 +1928,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2116,6 +2134,8 @@ define amdgpu_kernel void @local_wavefront_acquire_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2320,6 +2340,8 @@ define amdgpu_kernel void @local_wavefront_release_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2524,6 +2546,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2728,6 +2752,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2974,6 +3000,8 @@ define amdgpu_kernel void @local_wavefront_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3224,6 +3252,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3474,6 +3504,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3734,6 +3766,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3995,6 +4029,8 @@ define amdgpu_kernel void @local_wavefront_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4256,6 +4292,8 @@ define amdgpu_kernel void @local_wavefront_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4517,6 +4555,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4778,6 +4818,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5039,6 +5081,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5300,6 +5344,8 @@ define amdgpu_kernel void @local_wavefront_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5561,6 +5607,8 @@ define amdgpu_kernel void @local_wavefront_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5822,6 +5870,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6083,6 +6133,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6344,6 +6396,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6605,6 +6659,8 @@ define amdgpu_kernel void @local_wavefront_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6866,6 +6922,8 @@ define amdgpu_kernel void @local_wavefront_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7127,6 +7185,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7388,6 +7448,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7691,6 +7753,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7999,6 +8063,8 @@ define amdgpu_kernel void @local_wavefront_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8307,6 +8373,8 @@ define amdgpu_kernel void @local_wavefront_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8615,6 +8683,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8923,6 +8993,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9231,6 +9303,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9539,6 +9613,8 @@ define amdgpu_kernel void @local_wavefront_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9847,6 +9923,8 @@ define amdgpu_kernel void @local_wavefront_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10155,6 +10233,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10463,6 +10543,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10771,6 +10853,8 @@ define amdgpu_kernel void @local_wavefront_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11079,6 +11163,8 @@ define amdgpu_kernel void @local_wavefront_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11387,6 +11473,8 @@ define amdgpu_kernel void @local_wavefront_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11695,6 +11783,8 @@ define amdgpu_kernel void @local_wavefront_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12003,6 +12093,8 @@ define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12234,6 +12326,8 @@ define amdgpu_kernel void @local_wavefront_one_as_unordered_load(
;
; GFX1250-LABEL: local_wavefront_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12458,6 +12552,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_load(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12682,6 +12778,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_load(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12906,6 +13004,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_load(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13113,6 +13213,8 @@ define amdgpu_kernel void @local_wavefront_one_as_unordered_store(
;
; GFX1250-LABEL: local_wavefront_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13317,6 +13419,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_store(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13521,6 +13625,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_store(
;
; GFX1250-LABEL: local_wavefront_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13725,6 +13831,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_store(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13929,6 +14037,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14133,6 +14243,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14337,6 +14449,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14541,6 +14655,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14745,6 +14861,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14991,6 +15109,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15241,6 +15361,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15491,6 +15613,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15751,6 +15875,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16012,6 +16138,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16273,6 +16401,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16534,6 +16664,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16795,6 +16927,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17056,6 +17190,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17317,6 +17453,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17578,6 +17716,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17839,6 +17979,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18100,6 +18242,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18361,6 +18505,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18622,6 +18768,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18883,6 +19031,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19144,6 +19294,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19405,6 +19557,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19708,6 +19862,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_monotonic_ret_cmpxch
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20016,6 +20172,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20324,6 +20482,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20632,6 +20792,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20940,6 +21102,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21248,6 +21412,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21556,6 +21722,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21864,6 +22032,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22172,6 +22342,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22480,6 +22652,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22788,6 +22962,8 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23096,6 +23272,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23404,6 +23582,8 @@ define amdgpu_kernel void @local_wavefront_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23712,6 +23892,8 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24020,6 +24202,8 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
index 76b652dc3257d..b63c4236676f6 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
@@ -217,6 +217,8 @@ define amdgpu_kernel void @local_workgroup_unordered_load(
;
; GFX1250-LABEL: local_workgroup_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -441,6 +443,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_load(
;
; GFX1250-LABEL: local_workgroup_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -670,6 +674,8 @@ define amdgpu_kernel void @local_workgroup_acquire_load(
;
; GFX1250-LABEL: local_workgroup_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -922,6 +928,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_load(
;
; GFX1250-LABEL: local_workgroup_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1131,6 +1139,8 @@ define amdgpu_kernel void @local_workgroup_unordered_store(
;
; GFX1250-LABEL: local_workgroup_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1335,6 +1345,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_store(
;
; GFX1250-LABEL: local_workgroup_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1562,6 +1574,8 @@ define amdgpu_kernel void @local_workgroup_release_store(
;
; GFX1250-LABEL: local_workgroup_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1791,6 +1805,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_store(
;
; GFX1250-LABEL: local_workgroup_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1997,6 +2013,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2217,6 +2235,8 @@ define amdgpu_kernel void @local_workgroup_acquire_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2445,6 +2465,8 @@ define amdgpu_kernel void @local_workgroup_release_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2690,6 +2712,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2936,6 +2960,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3190,6 +3216,8 @@ define amdgpu_kernel void @local_workgroup_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3468,6 +3496,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3748,6 +3778,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4010,6 +4042,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4287,6 +4321,8 @@ define amdgpu_kernel void @local_workgroup_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4572,6 +4608,8 @@ define amdgpu_kernel void @local_workgroup_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4874,6 +4912,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5177,6 +5217,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5457,6 +5499,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5735,6 +5779,8 @@ define amdgpu_kernel void @local_workgroup_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6036,6 +6082,8 @@ define amdgpu_kernel void @local_workgroup_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6339,6 +6387,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6642,6 +6692,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6945,6 +6997,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7248,6 +7302,8 @@ define amdgpu_kernel void @local_workgroup_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7551,6 +7607,8 @@ define amdgpu_kernel void @local_workgroup_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7854,6 +7912,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8157,6 +8217,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8463,6 +8525,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8776,6 +8840,8 @@ define amdgpu_kernel void @local_workgroup_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9107,6 +9173,8 @@ define amdgpu_kernel void @local_workgroup_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9445,6 +9513,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9783,6 +9853,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10098,6 +10170,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10411,6 +10485,8 @@ define amdgpu_kernel void @local_workgroup_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10747,6 +10823,8 @@ define amdgpu_kernel void @local_workgroup_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11085,6 +11163,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11423,6 +11503,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11761,6 +11843,8 @@ define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12099,6 +12183,8 @@ define amdgpu_kernel void @local_workgroup_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12437,6 +12523,8 @@ define amdgpu_kernel void @local_workgroup_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12775,6 +12863,8 @@ define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13113,6 +13203,8 @@ define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13346,6 +13438,8 @@ define amdgpu_kernel void @local_workgroup_one_as_unordered_load(
;
; GFX1250-LABEL: local_workgroup_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13570,6 +13664,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_load(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13794,6 +13890,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_load(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14018,6 +14116,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_load(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14225,6 +14325,8 @@ define amdgpu_kernel void @local_workgroup_one_as_unordered_store(
;
; GFX1250-LABEL: local_workgroup_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14429,6 +14531,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_store(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14633,6 +14737,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_store(
;
; GFX1250-LABEL: local_workgroup_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14837,6 +14943,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_store(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15041,6 +15149,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15245,6 +15355,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15449,6 +15561,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15653,6 +15767,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15857,6 +15973,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16103,6 +16221,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16353,6 +16473,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16603,6 +16725,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16863,6 +16987,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17124,6 +17250,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17385,6 +17513,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17646,6 +17776,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17907,6 +18039,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18168,6 +18302,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18429,6 +18565,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18690,6 +18828,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18951,6 +19091,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19212,6 +19354,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19473,6 +19617,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19734,6 +19880,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19995,6 +20143,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20256,6 +20406,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20517,6 +20669,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20820,6 +20974,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_ret_cmpxch
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21128,6 +21284,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21436,6 +21594,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21744,6 +21904,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22052,6 +22214,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22360,6 +22524,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22668,6 +22834,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22976,6 +23144,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23284,6 +23454,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23592,6 +23764,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23900,6 +24074,8 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24208,6 +24384,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24516,6 +24694,8 @@ define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24824,6 +25004,8 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25132,6 +25314,8 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-agent.ll
index 3dee341ced5a2..840e75ebd011c 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-agent.ll
@@ -227,6 +227,8 @@ define amdgpu_kernel void @private_agent_unordered_load(
;
; GFX1250-LABEL: private_agent_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -480,6 +482,8 @@ define amdgpu_kernel void @private_agent_monotonic_load(
;
; GFX1250-LABEL: private_agent_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -733,6 +737,8 @@ define amdgpu_kernel void @private_agent_acquire_load(
;
; GFX1250-LABEL: private_agent_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -988,6 +994,8 @@ define amdgpu_kernel void @private_agent_seq_cst_load(
;
; GFX1250-LABEL: private_agent_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1229,6 +1237,8 @@ define amdgpu_kernel void @private_agent_unordered_store(
;
; GFX1250-LABEL: private_agent_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1464,6 +1474,8 @@ define amdgpu_kernel void @private_agent_monotonic_store(
;
; GFX1250-LABEL: private_agent_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1699,6 +1711,8 @@ define amdgpu_kernel void @private_agent_release_store(
;
; GFX1250-LABEL: private_agent_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1939,6 +1953,8 @@ define amdgpu_kernel void @private_agent_seq_cst_store(
;
; GFX1250-LABEL: private_agent_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2212,6 +2228,8 @@ define amdgpu_kernel void @private_agent_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_agent_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2481,6 +2499,8 @@ define amdgpu_kernel void @private_agent_acquire_atomicrmw(
;
; GFX1250-LABEL: private_agent_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2753,6 +2773,8 @@ define amdgpu_kernel void @private_agent_release_atomicrmw(
;
; GFX1250-LABEL: private_agent_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3027,6 +3049,8 @@ define amdgpu_kernel void @private_agent_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_agent_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3304,6 +3328,8 @@ define amdgpu_kernel void @private_agent_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_agent_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3602,6 +3628,8 @@ define amdgpu_kernel void @private_agent_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_agent_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3896,6 +3924,8 @@ define amdgpu_kernel void @private_agent_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_agent_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4195,6 +4225,8 @@ define amdgpu_kernel void @private_agent_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_agent_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4550,6 +4582,8 @@ define amdgpu_kernel void @private_agent_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4905,6 +4939,8 @@ define amdgpu_kernel void @private_agent_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5263,6 +5299,8 @@ define amdgpu_kernel void @private_agent_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5623,6 +5661,8 @@ define amdgpu_kernel void @private_agent_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5986,6 +6026,8 @@ define amdgpu_kernel void @private_agent_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6349,6 +6391,8 @@ define amdgpu_kernel void @private_agent_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6707,6 +6751,8 @@ define amdgpu_kernel void @private_agent_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7065,6 +7111,8 @@ define amdgpu_kernel void @private_agent_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7428,6 +7476,8 @@ define amdgpu_kernel void @private_agent_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7791,6 +7841,8 @@ define amdgpu_kernel void @private_agent_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8154,6 +8206,8 @@ define amdgpu_kernel void @private_agent_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8517,6 +8571,8 @@ define amdgpu_kernel void @private_agent_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8880,6 +8936,8 @@ define amdgpu_kernel void @private_agent_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9243,6 +9301,8 @@ define amdgpu_kernel void @private_agent_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9606,6 +9666,8 @@ define amdgpu_kernel void @private_agent_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9997,6 +10059,8 @@ define amdgpu_kernel void @private_agent_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10383,6 +10447,8 @@ define amdgpu_kernel void @private_agent_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10771,6 +10837,8 @@ define amdgpu_kernel void @private_agent_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11162,6 +11230,8 @@ define amdgpu_kernel void @private_agent_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11555,6 +11625,8 @@ define amdgpu_kernel void @private_agent_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11948,6 +12020,8 @@ define amdgpu_kernel void @private_agent_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12336,6 +12410,8 @@ define amdgpu_kernel void @private_agent_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12724,6 +12800,8 @@ define amdgpu_kernel void @private_agent_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13117,6 +13195,8 @@ define amdgpu_kernel void @private_agent_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13510,6 +13590,8 @@ define amdgpu_kernel void @private_agent_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13903,6 +13985,8 @@ define amdgpu_kernel void @private_agent_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14296,6 +14380,8 @@ define amdgpu_kernel void @private_agent_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14689,6 +14775,8 @@ define amdgpu_kernel void @private_agent_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15082,6 +15170,8 @@ define amdgpu_kernel void @private_agent_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15475,6 +15565,8 @@ define amdgpu_kernel void @private_agent_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15746,6 +15838,8 @@ define amdgpu_kernel void @private_agent_one_as_unordered_load(
;
; GFX1250-LABEL: private_agent_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15999,6 +16093,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_load(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16252,6 +16348,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_load(
;
; GFX1250-LABEL: private_agent_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16507,6 +16605,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_load(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16748,6 +16848,8 @@ define amdgpu_kernel void @private_agent_one_as_unordered_store(
;
; GFX1250-LABEL: private_agent_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16983,6 +17085,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_store(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17218,6 +17322,8 @@ define amdgpu_kernel void @private_agent_one_as_release_store(
;
; GFX1250-LABEL: private_agent_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17458,6 +17564,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_store(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17731,6 +17839,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18000,6 +18110,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18272,6 +18384,8 @@ define amdgpu_kernel void @private_agent_one_as_release_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18546,6 +18660,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18823,6 +18939,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19121,6 +19239,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19415,6 +19535,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19714,6 +19836,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20069,6 +20193,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20424,6 +20550,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20782,6 +20910,8 @@ define amdgpu_kernel void @private_agent_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21142,6 +21272,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21505,6 +21637,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21868,6 +22002,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22226,6 +22362,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22584,6 +22722,8 @@ define amdgpu_kernel void @private_agent_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22947,6 +23087,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23310,6 +23452,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23673,6 +23817,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24036,6 +24182,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24399,6 +24547,8 @@ define amdgpu_kernel void @private_agent_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24762,6 +24912,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25125,6 +25277,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25516,6 +25670,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25902,6 +26058,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26290,6 +26448,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26683,6 +26843,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27076,6 +27238,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27464,6 +27628,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27852,6 +28018,8 @@ define amdgpu_kernel void @private_agent_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28245,6 +28413,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28638,6 +28808,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29031,6 +29203,8 @@ define amdgpu_kernel void @private_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29424,6 +29598,8 @@ define amdgpu_kernel void @private_agent_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29817,6 +29993,8 @@ define amdgpu_kernel void @private_agent_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30210,6 +30388,8 @@ define amdgpu_kernel void @private_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30603,6 +30783,8 @@ define amdgpu_kernel void @private_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-cluster.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-cluster.ll
index d53d31a5fe5b9..3f6993a8fa579 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-cluster.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-cluster.ll
@@ -227,6 +227,8 @@ define amdgpu_kernel void @private_cluster_unordered_load(
;
; GFX1250-LABEL: private_cluster_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -480,6 +482,8 @@ define amdgpu_kernel void @private_cluster_monotonic_load(
;
; GFX1250-LABEL: private_cluster_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -733,6 +737,8 @@ define amdgpu_kernel void @private_cluster_acquire_load(
;
; GFX1250-LABEL: private_cluster_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -987,6 +993,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_load(
;
; GFX1250-LABEL: private_cluster_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1227,6 +1235,8 @@ define amdgpu_kernel void @private_cluster_unordered_store(
;
; GFX1250-LABEL: private_cluster_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1462,6 +1472,8 @@ define amdgpu_kernel void @private_cluster_monotonic_store(
;
; GFX1250-LABEL: private_cluster_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1697,6 +1709,8 @@ define amdgpu_kernel void @private_cluster_release_store(
;
; GFX1250-LABEL: private_cluster_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1934,6 +1948,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_store(
;
; GFX1250-LABEL: private_cluster_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2204,6 +2220,8 @@ define amdgpu_kernel void @private_cluster_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_cluster_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2473,6 +2491,8 @@ define amdgpu_kernel void @private_cluster_acquire_atomicrmw(
;
; GFX1250-LABEL: private_cluster_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2744,6 +2764,8 @@ define amdgpu_kernel void @private_cluster_release_atomicrmw(
;
; GFX1250-LABEL: private_cluster_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3015,6 +3037,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_cluster_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3288,6 +3312,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_cluster_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3582,6 +3608,8 @@ define amdgpu_kernel void @private_cluster_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_cluster_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3875,6 +3903,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_cluster_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4170,6 +4200,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_cluster_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4521,6 +4553,8 @@ define amdgpu_kernel void @private_cluster_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4876,6 +4910,8 @@ define amdgpu_kernel void @private_cluster_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5233,6 +5269,8 @@ define amdgpu_kernel void @private_cluster_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5590,6 +5628,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5949,6 +5989,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6308,6 +6350,8 @@ define amdgpu_kernel void @private_cluster_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6665,6 +6709,8 @@ define amdgpu_kernel void @private_cluster_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7022,6 +7068,8 @@ define amdgpu_kernel void @private_cluster_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7381,6 +7429,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7740,6 +7790,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8099,6 +8151,8 @@ define amdgpu_kernel void @private_cluster_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8458,6 +8512,8 @@ define amdgpu_kernel void @private_cluster_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8817,6 +8873,8 @@ define amdgpu_kernel void @private_cluster_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9176,6 +9234,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9535,6 +9595,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9922,6 +9984,8 @@ define amdgpu_kernel void @private_cluster_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10308,6 +10372,8 @@ define amdgpu_kernel void @private_cluster_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10695,6 +10761,8 @@ define amdgpu_kernel void @private_cluster_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11083,6 +11151,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11472,6 +11542,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11861,6 +11933,8 @@ define amdgpu_kernel void @private_cluster_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12248,6 +12322,8 @@ define amdgpu_kernel void @private_cluster_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12635,6 +12711,8 @@ define amdgpu_kernel void @private_cluster_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13024,6 +13102,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13413,6 +13493,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13802,6 +13884,8 @@ define amdgpu_kernel void @private_cluster_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14191,6 +14275,8 @@ define amdgpu_kernel void @private_cluster_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14580,6 +14666,8 @@ define amdgpu_kernel void @private_cluster_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14969,6 +15057,8 @@ define amdgpu_kernel void @private_cluster_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15358,6 +15448,8 @@ define amdgpu_kernel void @private_cluster_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15625,6 +15717,8 @@ define amdgpu_kernel void @private_cluster_one_as_unordered_load(
;
; GFX1250-LABEL: private_cluster_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15878,6 +15972,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_load(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16131,6 +16227,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_load(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16386,6 +16484,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_load(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16627,6 +16727,8 @@ define amdgpu_kernel void @private_cluster_one_as_unordered_store(
;
; GFX1250-LABEL: private_cluster_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16862,6 +16964,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_store(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17097,6 +17201,8 @@ define amdgpu_kernel void @private_cluster_one_as_release_store(
;
; GFX1250-LABEL: private_cluster_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17334,6 +17440,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_store(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17604,6 +17712,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17873,6 +17983,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18144,6 +18256,8 @@ define amdgpu_kernel void @private_cluster_one_as_release_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18415,6 +18529,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18688,6 +18804,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18982,6 +19100,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19276,6 +19396,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19572,6 +19694,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19924,6 +20048,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20279,6 +20405,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20636,6 +20764,8 @@ define amdgpu_kernel void @private_cluster_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20993,6 +21123,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21352,6 +21484,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21711,6 +21845,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22068,6 +22204,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22425,6 +22563,8 @@ define amdgpu_kernel void @private_cluster_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22784,6 +22924,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23143,6 +23285,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23502,6 +23646,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23861,6 +24007,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24220,6 +24368,8 @@ define amdgpu_kernel void @private_cluster_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24579,6 +24729,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24938,6 +25090,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25325,6 +25479,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25711,6 +25867,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26099,6 +26257,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26489,6 +26649,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26879,6 +27041,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27267,6 +27431,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27655,6 +27821,8 @@ define amdgpu_kernel void @private_cluster_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28045,6 +28213,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28435,6 +28605,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28825,6 +28997,8 @@ define amdgpu_kernel void @private_cluster_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29215,6 +29389,8 @@ define amdgpu_kernel void @private_cluster_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29605,6 +29781,8 @@ define amdgpu_kernel void @private_cluster_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29995,6 +30173,8 @@ define amdgpu_kernel void @private_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30385,6 +30565,8 @@ define amdgpu_kernel void @private_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_cluster_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
index cc3883c7724e9..be5250c04865a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll
@@ -23,6 +23,8 @@ define amdgpu_kernel void @private_last_use_load_0(ptr addrspace(5) %in, ptr add
;
; GFX1250-LABEL: private_last_use_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -81,6 +83,8 @@ define amdgpu_kernel void @private_last_use_load_1(ptr addrspace(5) %in, ptr add
;
; GFX1250-LABEL: private_last_use_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
@@ -143,6 +147,8 @@ define amdgpu_kernel void @private_last_use_and_volatile_load(ptr addrspace(5) %
;
; GFX1250-LABEL: private_last_use_and_volatile_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -199,6 +205,8 @@ define amdgpu_kernel void @private_last_use_and_nontemporal_load(ptr addrspace(5
;
; GFX1250-LABEL: private_last_use_and_nontemporal_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index b361d312206f2..462e7e033d271 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -247,6 +247,8 @@ define amdgpu_kernel void @private_nontemporal_load_0(
;
; GFX1250-LABEL: private_nontemporal_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -554,6 +556,8 @@ define amdgpu_kernel void @private_nontemporal_load_1(
;
; GFX1250-LABEL: private_nontemporal_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
@@ -789,6 +793,8 @@ define amdgpu_kernel void @private_nontemporal_store_0(
;
; GFX1250-LABEL: private_nontemporal_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1070,6 +1076,8 @@ define amdgpu_kernel void @private_nontemporal_store_1(
;
; GFX1250-LABEL: private_nontemporal_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1330,6 +1338,8 @@ define amdgpu_kernel void @private_nontemporal_volatile_load(
;
; GFX1250-LABEL: private_nontemporal_volatile_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-singlethread.ll
index 3ca4f129df783..99720639d3df3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-singlethread.ll
@@ -227,6 +227,8 @@ define amdgpu_kernel void @private_singlethread_unordered_load(
;
; GFX1250-LABEL: private_singlethread_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -480,6 +482,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_load(
;
; GFX1250-LABEL: private_singlethread_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -733,6 +737,8 @@ define amdgpu_kernel void @private_singlethread_acquire_load(
;
; GFX1250-LABEL: private_singlethread_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -986,6 +992,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_load(
;
; GFX1250-LABEL: private_singlethread_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1223,6 +1231,8 @@ define amdgpu_kernel void @private_singlethread_unordered_store(
;
; GFX1250-LABEL: private_singlethread_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1458,6 +1468,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_store(
;
; GFX1250-LABEL: private_singlethread_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1693,6 +1705,8 @@ define amdgpu_kernel void @private_singlethread_release_store(
;
; GFX1250-LABEL: private_singlethread_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1928,6 +1942,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_store(
;
; GFX1250-LABEL: private_singlethread_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2196,6 +2212,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2465,6 +2483,8 @@ define amdgpu_kernel void @private_singlethread_acquire_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2734,6 +2754,8 @@ define amdgpu_kernel void @private_singlethread_release_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3003,6 +3025,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3272,6 +3296,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3562,6 +3588,8 @@ define amdgpu_kernel void @private_singlethread_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3854,6 +3882,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4146,6 +4176,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4494,6 +4526,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4849,6 +4883,8 @@ define amdgpu_kernel void @private_singlethread_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5204,6 +5240,8 @@ define amdgpu_kernel void @private_singlethread_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5559,6 +5597,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5914,6 +5954,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6269,6 +6311,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6624,6 +6668,8 @@ define amdgpu_kernel void @private_singlethread_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6979,6 +7025,8 @@ define amdgpu_kernel void @private_singlethread_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7334,6 +7382,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7689,6 +7739,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8044,6 +8096,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8399,6 +8453,8 @@ define amdgpu_kernel void @private_singlethread_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8754,6 +8810,8 @@ define amdgpu_kernel void @private_singlethread_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9109,6 +9167,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9464,6 +9524,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9847,6 +9909,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10233,6 +10297,8 @@ define amdgpu_kernel void @private_singlethread_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10619,6 +10685,8 @@ define amdgpu_kernel void @private_singlethread_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11005,6 +11073,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11391,6 +11461,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11777,6 +11849,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12163,6 +12237,8 @@ define amdgpu_kernel void @private_singlethread_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12549,6 +12625,8 @@ define amdgpu_kernel void @private_singlethread_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12935,6 +13013,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13321,6 +13401,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13707,6 +13789,8 @@ define amdgpu_kernel void @private_singlethread_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14093,6 +14177,8 @@ define amdgpu_kernel void @private_singlethread_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14479,6 +14565,8 @@ define amdgpu_kernel void @private_singlethread_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14865,6 +14953,8 @@ define amdgpu_kernel void @private_singlethread_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15251,6 +15341,8 @@ define amdgpu_kernel void @private_singlethread_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15515,6 +15607,8 @@ define amdgpu_kernel void @private_singlethread_one_as_unordered_load(
;
; GFX1250-LABEL: private_singlethread_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15768,6 +15862,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_load(
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16021,6 +16117,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_load(
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16274,6 +16372,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_load(
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16511,6 +16611,8 @@ define amdgpu_kernel void @private_singlethread_one_as_unordered_store(
;
; GFX1250-LABEL: private_singlethread_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16746,6 +16848,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_store(
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16981,6 +17085,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_store(
;
; GFX1250-LABEL: private_singlethread_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17216,6 +17322,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_store(
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17484,6 +17592,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17753,6 +17863,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18022,6 +18134,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18291,6 +18405,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18560,6 +18676,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18850,6 +18968,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19142,6 +19262,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19434,6 +19556,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19782,6 +19906,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_monotonic_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20137,6 +20263,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_monotonic_cmpxchg
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20492,6 +20620,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_monotonic_cmpxchg
;
; GFX1250-LABEL: private_singlethread_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20847,6 +20977,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_monotonic_cmpxchg
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21202,6 +21334,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_monotonic_cmpxchg
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21557,6 +21691,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_acquire_cmpxchg
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21912,6 +22048,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22267,6 +22405,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22622,6 +22762,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22977,6 +23119,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23332,6 +23476,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_seq_cst_cmpxchg
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23687,6 +23833,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24042,6 +24190,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24397,6 +24547,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24752,6 +24904,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25135,6 +25289,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_monotonic_ret_c
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25521,6 +25677,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_monotonic_ret_cmp
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25907,6 +26065,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_monotonic_ret_cmp
;
; GFX1250-LABEL: private_singlethread_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26293,6 +26453,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_monotonic_ret_cmp
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26679,6 +26841,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_monotonic_ret_cmp
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27065,6 +27229,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_acquire_ret_cmp
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27451,6 +27617,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_acquire_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27837,6 +28005,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_acquire_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28223,6 +28393,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_acquire_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28609,6 +28781,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_acquire_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28995,6 +29169,8 @@ define amdgpu_kernel void @private_singlethread_one_as_monotonic_seq_cst_ret_cmp
;
; GFX1250-LABEL: private_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29381,6 +29557,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acquire_seq_cst_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29767,6 +29945,8 @@ define amdgpu_kernel void @private_singlethread_one_as_release_seq_cst_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30153,6 +30333,8 @@ define amdgpu_kernel void @private_singlethread_one_as_acq_rel_seq_cst_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30539,6 +30721,8 @@ define amdgpu_kernel void @private_singlethread_one_as_seq_cst_seq_cst_ret_cmpxc
;
; GFX1250-LABEL: private_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-system.ll
index 5bbeae134a765..1ff96eefdd446 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-system.ll
@@ -227,6 +227,8 @@ define amdgpu_kernel void @private_system_unordered_load(
;
; GFX1250-LABEL: private_system_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -480,6 +482,8 @@ define amdgpu_kernel void @private_system_monotonic_load(
;
; GFX1250-LABEL: private_system_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -733,6 +737,8 @@ define amdgpu_kernel void @private_system_acquire_load(
;
; GFX1250-LABEL: private_system_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -988,6 +994,8 @@ define amdgpu_kernel void @private_system_seq_cst_load(
;
; GFX1250-LABEL: private_system_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1229,6 +1237,8 @@ define amdgpu_kernel void @private_system_unordered_store(
;
; GFX1250-LABEL: private_system_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1464,6 +1474,8 @@ define amdgpu_kernel void @private_system_monotonic_store(
;
; GFX1250-LABEL: private_system_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1699,6 +1711,8 @@ define amdgpu_kernel void @private_system_release_store(
;
; GFX1250-LABEL: private_system_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1939,6 +1953,8 @@ define amdgpu_kernel void @private_system_seq_cst_store(
;
; GFX1250-LABEL: private_system_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2212,6 +2228,8 @@ define amdgpu_kernel void @private_system_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_system_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2481,6 +2499,8 @@ define amdgpu_kernel void @private_system_acquire_atomicrmw(
;
; GFX1250-LABEL: private_system_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2753,6 +2773,8 @@ define amdgpu_kernel void @private_system_release_atomicrmw(
;
; GFX1250-LABEL: private_system_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3027,6 +3049,8 @@ define amdgpu_kernel void @private_system_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_system_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3304,6 +3328,8 @@ define amdgpu_kernel void @private_system_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_system_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3602,6 +3628,8 @@ define amdgpu_kernel void @private_system_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_system_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3896,6 +3924,8 @@ define amdgpu_kernel void @private_system_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_system_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4195,6 +4225,8 @@ define amdgpu_kernel void @private_system_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_system_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4550,6 +4582,8 @@ define amdgpu_kernel void @private_system_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4905,6 +4939,8 @@ define amdgpu_kernel void @private_system_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5263,6 +5299,8 @@ define amdgpu_kernel void @private_system_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5623,6 +5661,8 @@ define amdgpu_kernel void @private_system_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5986,6 +6026,8 @@ define amdgpu_kernel void @private_system_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6349,6 +6391,8 @@ define amdgpu_kernel void @private_system_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6707,6 +6751,8 @@ define amdgpu_kernel void @private_system_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7065,6 +7111,8 @@ define amdgpu_kernel void @private_system_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7428,6 +7476,8 @@ define amdgpu_kernel void @private_system_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7791,6 +7841,8 @@ define amdgpu_kernel void @private_system_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8154,6 +8206,8 @@ define amdgpu_kernel void @private_system_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_system_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8545,6 +8599,8 @@ define amdgpu_kernel void @private_system_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8931,6 +8987,8 @@ define amdgpu_kernel void @private_system_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9319,6 +9377,8 @@ define amdgpu_kernel void @private_system_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9712,6 +9772,8 @@ define amdgpu_kernel void @private_system_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10105,6 +10167,8 @@ define amdgpu_kernel void @private_system_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10493,6 +10557,8 @@ define amdgpu_kernel void @private_system_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10881,6 +10947,8 @@ define amdgpu_kernel void @private_system_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11274,6 +11342,8 @@ define amdgpu_kernel void @private_system_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11667,6 +11737,8 @@ define amdgpu_kernel void @private_system_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12060,6 +12132,8 @@ define amdgpu_kernel void @private_system_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12453,6 +12527,8 @@ define amdgpu_kernel void @private_system_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12846,6 +12922,8 @@ define amdgpu_kernel void @private_system_relese_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_relese_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13239,6 +13317,8 @@ define amdgpu_kernel void @private_system_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13632,6 +13712,8 @@ define amdgpu_kernel void @private_system_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13903,6 +13985,8 @@ define amdgpu_kernel void @private_system_one_as_unordered_load(
;
; GFX1250-LABEL: private_system_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14156,6 +14240,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_load(
;
; GFX1250-LABEL: private_system_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14409,6 +14495,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_load(
;
; GFX1250-LABEL: private_system_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14664,6 +14752,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_load(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14905,6 +14995,8 @@ define amdgpu_kernel void @private_system_one_as_unordered_store(
;
; GFX1250-LABEL: private_system_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15140,6 +15232,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_store(
;
; GFX1250-LABEL: private_system_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15375,6 +15469,8 @@ define amdgpu_kernel void @private_system_one_as_release_store(
;
; GFX1250-LABEL: private_system_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15615,6 +15711,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_store(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15888,6 +15986,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16157,6 +16257,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16429,6 +16531,8 @@ define amdgpu_kernel void @private_system_one_as_release_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16703,6 +16807,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16980,6 +17086,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17278,6 +17386,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17572,6 +17682,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17871,6 +17983,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18226,6 +18340,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18581,6 +18697,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18939,6 +19057,8 @@ define amdgpu_kernel void @private_system_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19299,6 +19419,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19662,6 +19784,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20025,6 +20149,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20383,6 +20509,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20741,6 +20869,8 @@ define amdgpu_kernel void @private_system_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21104,6 +21234,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21467,6 +21599,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21830,6 +21964,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22193,6 +22329,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22556,6 +22694,8 @@ define amdgpu_kernel void @private_system_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22919,6 +23059,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23282,6 +23424,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23673,6 +23817,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_monotonic_ret_cmpxchg
;
; GFX1250-LABEL: private_system_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24059,6 +24205,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24447,6 +24595,8 @@ define amdgpu_kernel void @private_system_one_as_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24838,6 +24988,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25231,6 +25383,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25624,6 +25778,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26012,6 +26168,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26400,6 +26558,8 @@ define amdgpu_kernel void @private_system_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26793,6 +26953,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27186,6 +27348,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27579,6 +27743,8 @@ define amdgpu_kernel void @private_system_one_as_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27972,6 +28138,8 @@ define amdgpu_kernel void @private_system_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28365,6 +28533,8 @@ define amdgpu_kernel void @private_system_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28758,6 +28928,8 @@ define amdgpu_kernel void @private_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29151,6 +29323,8 @@ define amdgpu_kernel void @private_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
index fd822659c3439..66442d8abdeee 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
@@ -188,6 +188,8 @@ define amdgpu_kernel void @private_volatile_load_0(
;
; GFX1250-LABEL: private_volatile_load_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -419,6 +421,8 @@ define amdgpu_kernel void @private_volatile_load_1(
;
; GFX1250-LABEL: private_volatile_load_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
@@ -616,6 +620,8 @@ define amdgpu_kernel void @private_volatile_store_0(
;
; GFX1250-LABEL: private_volatile_store_0:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -838,6 +844,8 @@ define amdgpu_kernel void @private_volatile_store_1(
;
; GFX1250-LABEL: private_volatile_store_1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-wavefront.ll
index 4eb1cae3108e3..010a5c5902a15 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-wavefront.ll
@@ -227,6 +227,8 @@ define amdgpu_kernel void @private_wavefront_unordered_load(
;
; GFX1250-LABEL: private_wavefront_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -480,6 +482,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_load(
;
; GFX1250-LABEL: private_wavefront_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -733,6 +737,8 @@ define amdgpu_kernel void @private_wavefront_acquire_load(
;
; GFX1250-LABEL: private_wavefront_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -986,6 +992,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_load(
;
; GFX1250-LABEL: private_wavefront_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1223,6 +1231,8 @@ define amdgpu_kernel void @private_wavefront_unordered_store(
;
; GFX1250-LABEL: private_wavefront_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1458,6 +1468,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_store(
;
; GFX1250-LABEL: private_wavefront_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1693,6 +1705,8 @@ define amdgpu_kernel void @private_wavefront_release_store(
;
; GFX1250-LABEL: private_wavefront_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1928,6 +1942,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_store(
;
; GFX1250-LABEL: private_wavefront_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2196,6 +2212,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2465,6 +2483,8 @@ define amdgpu_kernel void @private_wavefront_acquire_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2734,6 +2754,8 @@ define amdgpu_kernel void @private_wavefront_release_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3003,6 +3025,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3272,6 +3296,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3562,6 +3588,8 @@ define amdgpu_kernel void @private_wavefront_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3854,6 +3882,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4146,6 +4176,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4494,6 +4526,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4849,6 +4883,8 @@ define amdgpu_kernel void @private_wavefront_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5204,6 +5240,8 @@ define amdgpu_kernel void @private_wavefront_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5559,6 +5597,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5914,6 +5954,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6269,6 +6311,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6624,6 +6668,8 @@ define amdgpu_kernel void @private_wavefront_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6979,6 +7025,8 @@ define amdgpu_kernel void @private_wavefront_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7334,6 +7382,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7689,6 +7739,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8044,6 +8096,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8399,6 +8453,8 @@ define amdgpu_kernel void @private_wavefront_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8754,6 +8810,8 @@ define amdgpu_kernel void @private_wavefront_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9109,6 +9167,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9464,6 +9524,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9847,6 +9909,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10233,6 +10297,8 @@ define amdgpu_kernel void @private_wavefront_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10619,6 +10685,8 @@ define amdgpu_kernel void @private_wavefront_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11005,6 +11073,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11391,6 +11461,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11777,6 +11849,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12163,6 +12237,8 @@ define amdgpu_kernel void @private_wavefront_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12549,6 +12625,8 @@ define amdgpu_kernel void @private_wavefront_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12935,6 +13013,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13321,6 +13401,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13707,6 +13789,8 @@ define amdgpu_kernel void @private_wavefront_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14093,6 +14177,8 @@ define amdgpu_kernel void @private_wavefront_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14479,6 +14565,8 @@ define amdgpu_kernel void @private_wavefront_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14865,6 +14953,8 @@ define amdgpu_kernel void @private_wavefront_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15251,6 +15341,8 @@ define amdgpu_kernel void @private_wavefront_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15515,6 +15607,8 @@ define amdgpu_kernel void @private_wavefront_one_as_unordered_load(
;
; GFX1250-LABEL: private_wavefront_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15768,6 +15862,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_load(
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16021,6 +16117,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_load(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16274,6 +16372,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_load(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16511,6 +16611,8 @@ define amdgpu_kernel void @private_wavefront_one_as_unordered_store(
;
; GFX1250-LABEL: private_wavefront_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16746,6 +16848,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_store(
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16981,6 +17085,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_store(
;
; GFX1250-LABEL: private_wavefront_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17216,6 +17322,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_store(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17484,6 +17592,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17753,6 +17863,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18022,6 +18134,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18291,6 +18405,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18560,6 +18676,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18850,6 +18968,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19142,6 +19262,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19434,6 +19556,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19782,6 +19906,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20137,6 +20263,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20492,6 +20620,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20847,6 +20977,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21202,6 +21334,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21557,6 +21691,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21912,6 +22048,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22267,6 +22405,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22622,6 +22762,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22977,6 +23119,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23332,6 +23476,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23687,6 +23833,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24042,6 +24190,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24397,6 +24547,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24752,6 +24904,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25135,6 +25289,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_monotonic_ret_cmpx
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25521,6 +25677,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25907,6 +26065,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_wavefront_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26293,6 +26453,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26679,6 +26841,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27065,6 +27229,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_acquire_ret_cmpxch
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27451,6 +27617,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27837,6 +28005,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28223,6 +28393,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28609,6 +28781,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28995,6 +29169,8 @@ define amdgpu_kernel void @private_wavefront_one_as_monotonic_seq_cst_ret_cmpxch
;
; GFX1250-LABEL: private_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29381,6 +29557,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29767,6 +29945,8 @@ define amdgpu_kernel void @private_wavefront_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30153,6 +30333,8 @@ define amdgpu_kernel void @private_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30539,6 +30721,8 @@ define amdgpu_kernel void @private_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-workgroup.ll
index 73e40a901ef53..755c0171a3baa 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-workgroup.ll
@@ -227,6 +227,8 @@ define amdgpu_kernel void @private_workgroup_unordered_load(
;
; GFX1250-LABEL: private_workgroup_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -480,6 +482,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_load(
;
; GFX1250-LABEL: private_workgroup_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -733,6 +737,8 @@ define amdgpu_kernel void @private_workgroup_acquire_load(
;
; GFX1250-LABEL: private_workgroup_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -986,6 +992,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_load(
;
; GFX1250-LABEL: private_workgroup_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1225,6 +1233,8 @@ define amdgpu_kernel void @private_workgroup_unordered_store(
;
; GFX1250-LABEL: private_workgroup_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1460,6 +1470,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_store(
;
; GFX1250-LABEL: private_workgroup_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1695,6 +1707,8 @@ define amdgpu_kernel void @private_workgroup_release_store(
;
; GFX1250-LABEL: private_workgroup_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1932,6 +1946,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_store(
;
; GFX1250-LABEL: private_workgroup_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2202,6 +2218,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2471,6 +2489,8 @@ define amdgpu_kernel void @private_workgroup_acquire_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -2741,6 +2761,8 @@ define amdgpu_kernel void @private_workgroup_release_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3012,6 +3034,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3284,6 +3308,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3577,6 +3603,8 @@ define amdgpu_kernel void @private_workgroup_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3869,6 +3897,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4163,6 +4193,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4513,6 +4545,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -4868,6 +4902,8 @@ define amdgpu_kernel void @private_workgroup_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5224,6 +5260,8 @@ define amdgpu_kernel void @private_workgroup_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5581,6 +5619,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -5939,6 +5979,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6297,6 +6339,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -6653,6 +6697,8 @@ define amdgpu_kernel void @private_workgroup_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7009,6 +7055,8 @@ define amdgpu_kernel void @private_workgroup_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7367,6 +7415,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -7725,6 +7775,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8083,6 +8135,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8441,6 +8495,8 @@ define amdgpu_kernel void @private_workgroup_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -8799,6 +8855,8 @@ define amdgpu_kernel void @private_workgroup_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9157,6 +9215,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9515,6 +9575,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -9901,6 +9963,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10287,6 +10351,8 @@ define amdgpu_kernel void @private_workgroup_acquire_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -10673,6 +10739,8 @@ define amdgpu_kernel void @private_workgroup_release_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11061,6 +11129,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11449,6 +11519,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_monotonic_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -11837,6 +11909,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12223,6 +12297,8 @@ define amdgpu_kernel void @private_workgroup_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12609,6 +12685,8 @@ define amdgpu_kernel void @private_workgroup_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -12997,6 +13075,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13385,6 +13465,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -13773,6 +13855,8 @@ define amdgpu_kernel void @private_workgroup_monotonic_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14161,6 +14245,8 @@ define amdgpu_kernel void @private_workgroup_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14549,6 +14635,8 @@ define amdgpu_kernel void @private_workgroup_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -14937,6 +15025,8 @@ define amdgpu_kernel void @private_workgroup_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15325,6 +15415,8 @@ define amdgpu_kernel void @private_workgroup_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15591,6 +15683,8 @@ define amdgpu_kernel void @private_workgroup_one_as_unordered_load(
;
; GFX1250-LABEL: private_workgroup_one_as_unordered_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -15844,6 +15938,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_load(
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16097,6 +16193,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_load(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16350,6 +16448,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_load(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_load:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16589,6 +16689,8 @@ define amdgpu_kernel void @private_workgroup_one_as_unordered_store(
;
; GFX1250-LABEL: private_workgroup_one_as_unordered_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -16824,6 +16926,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_store(
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17059,6 +17163,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_store(
;
; GFX1250-LABEL: private_workgroup_one_as_release_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17296,6 +17402,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_store(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_store:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17566,6 +17674,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -17835,6 +17945,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18105,6 +18217,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_release_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18376,6 +18490,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18648,6 +18764,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -18941,6 +19059,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_ret_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19233,6 +19353,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_ret_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19527,6 +19649,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_ret_atomicrmw(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_ret_atomicrmw:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -19877,6 +20001,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20232,6 +20358,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20588,6 +20716,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_release_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -20945,6 +21075,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21303,6 +21435,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_monotonic_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_monotonic_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -21661,6 +21795,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22017,6 +22153,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22373,6 +22511,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_release_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -22731,6 +22871,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23089,6 +23231,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_acquire_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_acquire_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23447,6 +23591,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -23805,6 +23951,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24163,6 +24311,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_release_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24521,6 +24671,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -24879,6 +25031,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25265,6 +25419,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_monotonic_ret_cmpx
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -25651,6 +25807,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26037,6 +26195,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_workgroup_one_as_release_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26425,6 +26585,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -26813,6 +26975,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_monotonic_ret_cmpxch
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27201,6 +27365,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_acquire_ret_cmpxch
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27587,6 +27753,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -27973,6 +28141,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_release_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28361,6 +28531,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -28749,6 +28921,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29137,6 +29311,8 @@ define amdgpu_kernel void @private_workgroup_one_as_monotonic_seq_cst_ret_cmpxch
;
; GFX1250-LABEL: private_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29525,6 +29701,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -29913,6 +30091,8 @@ define amdgpu_kernel void @private_workgroup_one_as_release_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_release_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30301,6 +30481,8 @@ define amdgpu_kernel void @private_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -30689,6 +30871,8 @@ define amdgpu_kernel void @private_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
;
; GFX1250-LABEL: private_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index e24b465ff99ff..43839fe070cc6 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -129,6 +129,8 @@ define amdgpu_kernel void @v_test_imin_sle_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_imin_sle_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -227,6 +229,8 @@ define amdgpu_kernel void @s_test_imin_sle_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_imin_sle_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -315,6 +319,8 @@ define amdgpu_kernel void @s_test_imin_sle_v1i32(ptr addrspace(1) %out, <1 x i32
;
; GFX1250-LABEL: s_test_imin_sle_v1i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -442,6 +448,8 @@ define amdgpu_kernel void @s_test_imin_sle_v4i32(ptr addrspace(1) %out, <4 x i32
;
; GFX1250-LABEL: s_test_imin_sle_v4i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10 nv
@@ -578,6 +586,8 @@ define amdgpu_kernel void @s_test_imin_sle_i8(ptr addrspace(1) %out, [8 x i32],
;
; GFX1250-LABEL: s_test_imin_sle_i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
@@ -834,6 +844,8 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
;
; GFX1250-LABEL: s_test_imin_sle_v4i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
@@ -980,6 +992,8 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
;
; GFX1250-LABEL: s_test_imin_sle_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -1177,6 +1191,8 @@ define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16
;
; GFX1250-LABEL: s_test_imin_sle_v4i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
@@ -1313,6 +1329,8 @@ define amdgpu_kernel void @v_test_imin_slt_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_imin_slt_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -1491,6 +1509,8 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-TRUE16-LABEL: v_test_imin_slt_i16:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -1507,6 +1527,8 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-FAKE16-LABEL: v_test_imin_slt_i16:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -1606,6 +1628,8 @@ define amdgpu_kernel void @s_test_imin_slt_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_imin_slt_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -1711,6 +1735,8 @@ define amdgpu_kernel void @s_test_imin_slt_v2i32(ptr addrspace(1) %out, <2 x i32
;
; GFX1250-LABEL: s_test_imin_slt_v2i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
@@ -1809,6 +1835,8 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX1250-LABEL: s_test_imin_slt_imm_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -1904,6 +1932,8 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX1250-LABEL: s_test_imin_sle_imm_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -2039,6 +2069,8 @@ define amdgpu_kernel void @v_test_umin_ule_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_umin_ule_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -2198,6 +2230,8 @@ define amdgpu_kernel void @v_test_umin_ule_v3i32(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_umin_ule_v3i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -2402,6 +2436,8 @@ define amdgpu_kernel void @v_test_umin_ule_v3i16(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_umin_ule_v3i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -2506,6 +2542,8 @@ define amdgpu_kernel void @s_test_umin_ule_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_umin_ule_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -2641,6 +2679,8 @@ define amdgpu_kernel void @v_test_umin_ult_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_umin_ult_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -2802,6 +2842,8 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-TRUE16-LABEL: v_test_umin_ult_i8:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -2818,6 +2860,8 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-FAKE16-LABEL: v_test_umin_ult_i8:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -2917,6 +2961,8 @@ define amdgpu_kernel void @s_test_umin_ult_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_umin_ult_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -3066,6 +3112,8 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(ptr addrspace(1) %out0,
;
; GFX1250-LABEL: v_test_umin_ult_i32_multi_use:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -3261,6 +3309,8 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
;
; GFX1250-LABEL: v_test_umin_ult_i16_multi_use:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -3362,6 +3412,8 @@ define amdgpu_kernel void @s_test_umin_ult_v1i32(ptr addrspace(1) %out, <1 x i32
;
; GFX1250-LABEL: s_test_umin_ult_v1i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -3550,6 +3602,8 @@ define amdgpu_kernel void @s_test_umin_ult_v8i32(ptr addrspace(1) %out, <8 x i32
;
; GFX1250-LABEL: s_test_umin_ult_v8i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b512 s[8:23], s[4:5], 0x20 nv
@@ -3867,6 +3921,8 @@ define amdgpu_kernel void @s_test_umin_ult_v8i16(ptr addrspace(1) %out, <8 x i16
;
; GFX1250-LABEL: s_test_umin_ult_v8i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10 nv
@@ -3992,6 +4048,8 @@ define amdgpu_kernel void @simplify_demanded_bits_test_umin_ult_i16(ptr addrspac
;
; GFX1250-LABEL: simplify_demanded_bits_test_umin_ult_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
@@ -4122,6 +4180,8 @@ define amdgpu_kernel void @simplify_demanded_bits_test_min_slt_i16(ptr addrspace
;
; GFX1250-LABEL: simplify_demanded_bits_test_min_slt_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
@@ -4255,6 +4315,8 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
;
; GFX1250-LABEL: s_test_imin_sle_i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
@@ -4384,6 +4446,8 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_umin_ult_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -4509,6 +4573,8 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_umin_ule_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -4634,6 +4700,8 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_imin_slt_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -4759,6 +4827,8 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_imin_sle_i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -4922,6 +4992,8 @@ define amdgpu_kernel void @v_test_imin_sle_v2i16(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_imin_sle_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -5093,6 +5165,8 @@ define amdgpu_kernel void @v_test_imin_ule_v2i16(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_imin_ule_v2i16:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
diff --git a/llvm/test/CodeGen/AMDGPU/minmax.ll b/llvm/test/CodeGen/AMDGPU/minmax.ll
index baaaf657478c3..509b289fbb173 100644
--- a/llvm/test/CodeGen/AMDGPU/minmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/minmax.ll
@@ -113,6 +113,8 @@ define amdgpu_ps void @s_test_minmax_i32(i32 inreg %a, i32 inreg %b, i32 inreg %
;
; SDAG-GFX1250-LABEL: s_test_minmax_i32:
; SDAG-GFX1250: ; %bb.0:
+; SDAG-GFX1250-NEXT: global_wb
+; SDAG-GFX1250-NEXT: v_nop
; SDAG-GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-NEXT: s_max_i32 s0, s0, s1
; SDAG-GFX1250-NEXT: s_mov_b32 s5, s4
@@ -124,6 +126,8 @@ define amdgpu_ps void @s_test_minmax_i32(i32 inreg %a, i32 inreg %b, i32 inreg %
;
; GISEL-GFX1250-LABEL: s_test_minmax_i32:
; GISEL-GFX1250: ; %bb.0:
+; GISEL-GFX1250-NEXT: global_wb
+; GISEL-GFX1250-NEXT: v_nop
; GISEL-GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-NEXT: s_max_i32 s0, s0, s1
; GISEL-GFX1250-NEXT: s_mov_b32 s6, s3
@@ -378,6 +382,8 @@ define amdgpu_ps void @s_test_minmax_u32(i32 inreg %a, i32 inreg %b, i32 inreg %
;
; SDAG-GFX1250-LABEL: s_test_minmax_u32:
; SDAG-GFX1250: ; %bb.0:
+; SDAG-GFX1250-NEXT: global_wb
+; SDAG-GFX1250-NEXT: v_nop
; SDAG-GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-NEXT: s_max_u32 s0, s0, s1
; SDAG-GFX1250-NEXT: s_mov_b32 s5, s4
@@ -389,6 +395,8 @@ define amdgpu_ps void @s_test_minmax_u32(i32 inreg %a, i32 inreg %b, i32 inreg %
;
; GISEL-GFX1250-LABEL: s_test_minmax_u32:
; GISEL-GFX1250: ; %bb.0:
+; GISEL-GFX1250-NEXT: global_wb
+; GISEL-GFX1250-NEXT: v_nop
; GISEL-GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-NEXT: s_max_u32 s0, s0, s1
; GISEL-GFX1250-NEXT: s_mov_b32 s6, s3
@@ -686,6 +694,8 @@ define amdgpu_ps void @s_test_minmax_f32_ieee_false(float inreg %a, float inreg
;
; SDAG-GFX1250-LABEL: s_test_minmax_f32_ieee_false:
; SDAG-GFX1250: ; %bb.0:
+; SDAG-GFX1250-NEXT: global_wb
+; SDAG-GFX1250-NEXT: v_nop
; SDAG-GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
; SDAG-GFX1250-NEXT: s_mov_b32 s5, s4
@@ -696,6 +706,8 @@ define amdgpu_ps void @s_test_minmax_f32_ieee_false(float inreg %a, float inreg
;
; GISEL-GFX1250-LABEL: s_test_minmax_f32_ieee_false:
; GISEL-GFX1250: ; %bb.0:
+; GISEL-GFX1250-NEXT: global_wb
+; GISEL-GFX1250-NEXT: v_nop
; GISEL-GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-NEXT: s_max_num_f32 s0, s0, s1
; GISEL-GFX1250-NEXT: s_mov_b32 s6, s3
@@ -729,6 +741,8 @@ define amdgpu_ps float @test_minmax_commuted_f32_ieee_false(float %a, float %b,
;
; GFX1250-LABEL: test_minmax_commuted_f32_ieee_false:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_maxmin_num_f32 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
@@ -834,6 +848,8 @@ define amdgpu_ps float @test_maxmin_commuted_f32_ieee_false(float %a, float %b,
;
; GFX1250-LABEL: test_maxmin_commuted_f32_ieee_false:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_minmax_num_f32 v0, v0, v1, v2
; GFX1250-NEXT: ; return to shader part epilog
@@ -987,24 +1003,32 @@ define amdgpu_ps half @test_minmax_f16_ieee_false(half %a, half %b, half %c) {
;
; SDAG-GFX1250-TRUE16-LABEL: test_minmax_f16_ieee_false:
; SDAG-GFX1250-TRUE16: ; %bb.0:
+; SDAG-GFX1250-TRUE16-NEXT: global_wb
+; SDAG-GFX1250-TRUE16-NEXT: v_nop
; SDAG-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-TRUE16-NEXT: v_maxmin_num_f16 v0.l, v0.l, v1.l, v2.l
; SDAG-GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; SDAG-GFX1250-FAKE16-LABEL: test_minmax_f16_ieee_false:
; SDAG-GFX1250-FAKE16: ; %bb.0:
+; SDAG-GFX1250-FAKE16-NEXT: global_wb
+; SDAG-GFX1250-FAKE16-NEXT: v_nop
; SDAG-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-FAKE16-NEXT: v_maxmin_num_f16 v0, v0, v1, v2
; SDAG-GFX1250-FAKE16-NEXT: ; return to shader part epilog
;
; GISEL-GFX1250-TRUE16-LABEL: test_minmax_f16_ieee_false:
; GISEL-GFX1250-TRUE16: ; %bb.0:
+; GISEL-GFX1250-TRUE16-NEXT: global_wb
+; GISEL-GFX1250-TRUE16-NEXT: v_nop
; GISEL-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-TRUE16-NEXT: v_maxmin_num_f16 v0.l, v0.l, v1.l, v2.l
; GISEL-GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; GISEL-GFX1250-FAKE16-LABEL: test_minmax_f16_ieee_false:
; GISEL-GFX1250-FAKE16: ; %bb.0:
+; GISEL-GFX1250-FAKE16-NEXT: global_wb
+; GISEL-GFX1250-FAKE16-NEXT: v_nop
; GISEL-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-FAKE16-NEXT: v_maxmin_num_f16 v0, v0, v1, v2
; GISEL-GFX1250-FAKE16-NEXT: ; return to shader part epilog
@@ -1136,6 +1160,8 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
;
; SDAG-GFX1250-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
; SDAG-GFX1250-TRUE16: ; %bb.0:
+; SDAG-GFX1250-TRUE16-NEXT: global_wb
+; SDAG-GFX1250-TRUE16-NEXT: v_nop
; SDAG-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; SDAG-GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -1147,6 +1173,8 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
;
; SDAG-GFX1250-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
; SDAG-GFX1250-FAKE16: ; %bb.0:
+; SDAG-GFX1250-FAKE16-NEXT: global_wb
+; SDAG-GFX1250-FAKE16-NEXT: v_nop
; SDAG-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 0
; SDAG-GFX1250-FAKE16-NEXT: s_mov_b32 s5, s4
@@ -1157,6 +1185,8 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
;
; GISEL-GFX1250-TRUE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX1250-TRUE16: ; %bb.0:
+; GISEL-GFX1250-TRUE16-NEXT: global_wb
+; GISEL-GFX1250-TRUE16-NEXT: v_nop
; GISEL-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-TRUE16-NEXT: s_max_num_f16 s0, s0, s1
; GISEL-GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
@@ -1169,6 +1199,8 @@ define amdgpu_ps void @s_test_minmax_f16_ieee_false(half inreg %a, half inreg %b
;
; GISEL-GFX1250-FAKE16-LABEL: s_test_minmax_f16_ieee_false:
; GISEL-GFX1250-FAKE16: ; %bb.0:
+; GISEL-GFX1250-FAKE16-NEXT: global_wb
+; GISEL-GFX1250-FAKE16-NEXT: v_nop
; GISEL-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-FAKE16-NEXT: s_max_num_f16 s0, s0, s1
; GISEL-GFX1250-FAKE16-NEXT: s_mov_b32 s6, s3
@@ -1416,24 +1448,32 @@ define amdgpu_ps half @test_maxmin_f16_ieee_false(half %a, half %b, half %c) {
;
; SDAG-GFX1250-TRUE16-LABEL: test_maxmin_f16_ieee_false:
; SDAG-GFX1250-TRUE16: ; %bb.0:
+; SDAG-GFX1250-TRUE16-NEXT: global_wb
+; SDAG-GFX1250-TRUE16-NEXT: v_nop
; SDAG-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-TRUE16-NEXT: v_minmax_num_f16 v0.l, v0.l, v1.l, v2.l
; SDAG-GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; SDAG-GFX1250-FAKE16-LABEL: test_maxmin_f16_ieee_false:
; SDAG-GFX1250-FAKE16: ; %bb.0:
+; SDAG-GFX1250-FAKE16-NEXT: global_wb
+; SDAG-GFX1250-FAKE16-NEXT: v_nop
; SDAG-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-GFX1250-FAKE16-NEXT: v_minmax_num_f16 v0, v0, v1, v2
; SDAG-GFX1250-FAKE16-NEXT: ; return to shader part epilog
;
; GISEL-GFX1250-TRUE16-LABEL: test_maxmin_f16_ieee_false:
; GISEL-GFX1250-TRUE16: ; %bb.0:
+; GISEL-GFX1250-TRUE16-NEXT: global_wb
+; GISEL-GFX1250-TRUE16-NEXT: v_nop
; GISEL-GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-TRUE16-NEXT: v_minmax_num_f16 v0.l, v0.l, v1.l, v2.l
; GISEL-GFX1250-TRUE16-NEXT: ; return to shader part epilog
;
; GISEL-GFX1250-FAKE16-LABEL: test_maxmin_f16_ieee_false:
; GISEL-GFX1250-FAKE16: ; %bb.0:
+; GISEL-GFX1250-FAKE16-NEXT: global_wb
+; GISEL-GFX1250-FAKE16-NEXT: v_nop
; GISEL-GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-GFX1250-FAKE16-NEXT: v_minmax_num_f16 v0, v0, v1, v2
; GISEL-GFX1250-FAKE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index cb18594654611..4c16159273930 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -128,6 +128,8 @@ define amdgpu_kernel void @test_mul_v2i32(ptr addrspace(1) %out, ptr addrspace(1
;
; GFX1250-LABEL: test_mul_v2i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -329,6 +331,8 @@ define amdgpu_kernel void @v_mul_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %
;
; GFX1250-LABEL: v_mul_v4i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -492,6 +496,8 @@ define amdgpu_kernel void @s_trunc_i64_mul_to_i32(ptr addrspace(1) %out, i64 %a,
;
; GFX1250-LABEL: s_trunc_i64_mul_to_i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -673,6 +679,8 @@ define amdgpu_kernel void @v_trunc_i64_mul_to_i32(ptr addrspace(1) %out, ptr add
;
; GFX1250-LABEL: v_trunc_i64_mul_to_i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -835,6 +843,8 @@ define amdgpu_kernel void @mul64_sext_c(ptr addrspace(1) %out, i32 %in) {
;
; GFX1250-LABEL: mul64_sext_c:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -965,6 +975,8 @@ define amdgpu_kernel void @mul64_zext_c(ptr addrspace(1) %out, i32 %in) {
;
; GFX1250-LABEL: mul64_zext_c:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s3, 0
@@ -1126,6 +1138,8 @@ define amdgpu_kernel void @v_mul64_sext_c(ptr addrspace(1) %out, ptr addrspace(1
;
; GFX1250-LABEL: v_mul64_sext_c:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -1309,6 +1323,8 @@ define amdgpu_kernel void @v_mul64_zext_c(ptr addrspace(1) %out, ptr addrspace(1
;
; GFX1250-LABEL: v_mul64_zext_c:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -1488,6 +1504,8 @@ define amdgpu_kernel void @v_mul64_sext_inline_imm(ptr addrspace(1) %out, ptr ad
;
; GFX1250-LABEL: v_mul64_sext_inline_imm:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -1636,6 +1654,8 @@ define amdgpu_kernel void @s_mul_i32(ptr addrspace(1) %out, [8 x i32], i32 %a, [
;
; GFX1250-LABEL: s_mul_i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
@@ -1790,6 +1810,8 @@ define amdgpu_kernel void @v_mul_i32(ptr addrspace(1) %out, ptr addrspace(1) %in
;
; GFX1250-LABEL: v_mul_i32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -1942,6 +1964,8 @@ define amdgpu_kernel void @s_mul_i1(ptr addrspace(1) %out, [8 x i32], i1 %a, [8
;
; GFX1250-LABEL: s_mul_i1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
@@ -2134,6 +2158,8 @@ define amdgpu_kernel void @v_mul_i1(ptr addrspace(1) %out, ptr addrspace(1) %in)
;
; GFX1250-LABEL: v_mul_i1:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, -1
@@ -2336,6 +2362,8 @@ define amdgpu_kernel void @s_mul_i64(ptr addrspace(1) %out, i64 %a, i64 %b) noun
;
; GFX1250-LABEL: s_mul_i64:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -2551,6 +2579,8 @@ define amdgpu_kernel void @v_mul_i64(ptr addrspace(1) %out, ptr addrspace(1) %ap
;
; GFX1250-LABEL: v_mul_i64:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -2860,6 +2890,8 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1250-LABEL: mul32_in_branch:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
; GFX1250-NEXT: s_mov_b32 s7, 0
@@ -3223,6 +3255,8 @@ define amdgpu_kernel void @mul64_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1250-LABEL: mul64_in_branch:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -3636,6 +3670,8 @@ define amdgpu_kernel void @s_mul_i128(ptr addrspace(1) %out, [8 x i32], i128 %a,
;
; GFX1250-LABEL: s_mul_i128:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
; GFX1250-NEXT: s_load_b128 s[8:11], s[4:5], 0x7c nv
@@ -4003,6 +4039,8 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
;
; GFX1250-LABEL: v_mul_i128:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
; GFX1250-NEXT: v_and_b32_e32 v16, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index dca6ad5efa859..6d75d3baa8eb1 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -34,6 +34,8 @@ define amdgpu_kernel void @fadd_v2_vv(ptr addrspace(1) %a) {
;
; GFX1250-LABEL: fadd_v2_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -78,6 +80,8 @@ define amdgpu_kernel void @fadd_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
;
; GFX1250-LABEL: fadd_v2_vs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -143,6 +147,8 @@ define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
;
; GFX1250-SDAG-LABEL: fadd_v4_vs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -161,6 +167,8 @@ define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
;
; GFX1250-GISEL-LABEL: fadd_v4_vs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_clause 0x1
; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -347,6 +355,8 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
;
; GFX1250-SDAG-LABEL: fadd_v32_vs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -414,6 +424,8 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
;
; GFX1250-GISEL-LABEL: fadd_v32_vs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -533,6 +545,8 @@ define amdgpu_kernel void @fadd_v2_v_imm(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_imm:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -546,6 +560,8 @@ define amdgpu_kernel void @fadd_v2_v_imm(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_imm:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -607,6 +623,8 @@ define amdgpu_kernel void @fadd_v2_v_v_splat(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_v_splat:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -619,6 +637,8 @@ define amdgpu_kernel void @fadd_v2_v_v_splat(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_v_splat:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -682,6 +702,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_splat(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_lit_splat:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -694,6 +716,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_splat(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_lit_splat:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -743,6 +767,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_hi0(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_lit_hi0:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -756,6 +782,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_hi0(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_lit_hi0:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -805,6 +833,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_lo0(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_lit_lo0:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -818,6 +848,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_lo0(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_lit_lo0:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -867,6 +899,8 @@ define amdgpu_kernel void @fadd_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_unfoldable_lit:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -880,6 +914,8 @@ define amdgpu_kernel void @fadd_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_unfoldable_lit:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -945,6 +981,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -957,6 +995,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_fneg:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1041,6 +1081,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_lo:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1053,6 +1095,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_fneg_lo:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1140,6 +1184,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, float %x) {
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_hi:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1152,6 +1198,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, float %x) {
;
; GFX1250-GISEL-LABEL: fadd_v2_v_fneg_hi:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1234,6 +1282,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_lo2:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1248,6 +1298,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
;
; GFX1250-GISEL-LABEL: fadd_v2_v_fneg_lo2:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1331,6 +1383,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
;
; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_hi2:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1345,6 +1399,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
;
; GFX1250-GISEL-LABEL: fadd_v2_v_fneg_hi2:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1398,6 +1454,8 @@ define amdgpu_kernel void @fmul_v2_vv(ptr addrspace(1) %a) {
;
; GFX1250-LABEL: fmul_v2_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1442,6 +1500,8 @@ define amdgpu_kernel void @fmul_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
;
; GFX1250-LABEL: fmul_v2_vs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1507,6 +1567,8 @@ define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
;
; GFX1250-SDAG-LABEL: fmul_v4_vs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -1525,6 +1587,8 @@ define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
;
; GFX1250-GISEL-LABEL: fmul_v4_vs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_clause 0x1
; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -1711,6 +1775,8 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
;
; GFX1250-SDAG-LABEL: fmul_v32_vs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1778,6 +1844,8 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
;
; GFX1250-GISEL-LABEL: fmul_v32_vs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1896,6 +1964,8 @@ define amdgpu_kernel void @fmul_v2_v_imm(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fmul_v2_v_imm:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -1909,6 +1979,8 @@ define amdgpu_kernel void @fmul_v2_v_imm(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fmul_v2_v_imm:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1970,6 +2042,8 @@ define amdgpu_kernel void @fmul_v2_v_v_splat(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fmul_v2_v_v_splat:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1982,6 +2056,8 @@ define amdgpu_kernel void @fmul_v2_v_v_splat(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fmul_v2_v_v_splat:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2045,6 +2121,8 @@ define amdgpu_kernel void @fmul_v2_v_lit_splat(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fmul_v2_v_lit_splat:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -2057,6 +2135,8 @@ define amdgpu_kernel void @fmul_v2_v_lit_splat(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fmul_v2_v_lit_splat:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -2107,6 +2187,8 @@ define amdgpu_kernel void @fmul_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fmul_v2_v_unfoldable_lit:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -2120,6 +2202,8 @@ define amdgpu_kernel void @fmul_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fmul_v2_v_unfoldable_lit:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -2184,6 +2268,8 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; GFX1250-SDAG-LABEL: fmul_v2_v_fneg:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -2196,6 +2282,8 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; GFX1250-GISEL-LABEL: fmul_v2_v_fneg:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -2246,6 +2334,8 @@ define amdgpu_kernel void @fma_v2_vv(ptr addrspace(1) %a) {
;
; GFX1250-LABEL: fma_v2_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -2290,6 +2380,8 @@ define amdgpu_kernel void @fma_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
;
; GFX1250-LABEL: fma_v2_vs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -2355,6 +2447,8 @@ define amdgpu_kernel void @fma_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
;
; GFX1250-SDAG-LABEL: fma_v4_vs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -2373,6 +2467,8 @@ define amdgpu_kernel void @fma_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
;
; GFX1250-GISEL-LABEL: fma_v4_vs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_clause 0x1
; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -2559,6 +2655,8 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
;
; GFX1250-SDAG-LABEL: fma_v32_vs:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2626,6 +2724,8 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
;
; GFX1250-GISEL-LABEL: fma_v32_vs:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2767,6 +2867,8 @@ define amdgpu_kernel void @fma_v2_v_imm(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fma_v2_v_imm:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -2782,6 +2884,8 @@ define amdgpu_kernel void @fma_v2_v_imm(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fma_v2_v_imm:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v6, 0x3ff, v0
@@ -2846,6 +2950,8 @@ define amdgpu_kernel void @fma_v2_v_v_splat(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fma_v2_v_v_splat:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2858,6 +2964,8 @@ define amdgpu_kernel void @fma_v2_v_v_splat(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fma_v2_v_v_splat:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2941,6 +3049,8 @@ define amdgpu_kernel void @fma_v2_v_lit_splat(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fma_v2_v_lit_splat:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -2953,6 +3063,8 @@ define amdgpu_kernel void @fma_v2_v_lit_splat(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fma_v2_v_lit_splat:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v6, 0x3ff, v0
@@ -3043,6 +3155,8 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fma_v2_v_unfoldable_lit:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v6, 0x3ff, v0
@@ -3057,6 +3171,8 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fma_v2_v_unfoldable_lit:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v6, 0x3ff, v0
@@ -3123,6 +3239,8 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; GFX1250-SDAG-LABEL: fma_v2_v_fneg:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -3135,6 +3253,8 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; GFX1250-GISEL-LABEL: fma_v2_v_fneg:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -3206,6 +3326,8 @@ define amdgpu_kernel void @add_vector_neg_bitcast_scalar_lo(ptr addrspace(1) %ou
;
; GFX1250-SDAG-LABEL: add_vector_neg_bitcast_scalar_lo:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -3220,6 +3342,8 @@ define amdgpu_kernel void @add_vector_neg_bitcast_scalar_lo(ptr addrspace(1) %ou
;
; GFX1250-GISEL-LABEL: add_vector_neg_bitcast_scalar_lo:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -3303,6 +3427,8 @@ define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspa
;
; GFX1250-SDAG-LABEL: fma_vector_vector_neg_scalar_lo_scalar_hi:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -3319,6 +3445,8 @@ define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspa
;
; GFX1250-GISEL-LABEL: fma_vector_vector_neg_scalar_lo_scalar_hi:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -3418,6 +3546,8 @@ define amdgpu_kernel void @shuffle_add_f32(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1250-SDAG-LABEL: shuffle_add_f32:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -3431,6 +3561,8 @@ define amdgpu_kernel void @shuffle_add_f32(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1250-GISEL-LABEL: shuffle_add_f32:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -3536,6 +3668,8 @@ define amdgpu_kernel void @shuffle_neg_add_f32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-SDAG-LABEL: shuffle_neg_add_f32:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -3552,6 +3686,8 @@ define amdgpu_kernel void @shuffle_neg_add_f32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-GISEL-LABEL: shuffle_neg_add_f32:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -3635,6 +3771,8 @@ define amdgpu_kernel void @fadd_fadd_fsub_0(<2 x float> %arg) {
;
; GFX1250-SDAG-LABEL: fadd_fadd_fsub_0:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -3647,6 +3785,8 @@ define amdgpu_kernel void @fadd_fadd_fsub_0(<2 x float> %arg) {
;
; GFX1250-GISEL-LABEL: fadd_fadd_fsub_0:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -3743,6 +3883,8 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
;
; GFX1250-SDAG-LABEL: fadd_fadd_fsub:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -3761,6 +3903,8 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
;
; GFX1250-GISEL-LABEL: fadd_fadd_fsub:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_clause 0x1
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -3837,6 +3981,8 @@ define amdgpu_kernel void @fadd_shuffle_v4(ptr addrspace(1) %arg) {
;
; GFX1250-SDAG-LABEL: fadd_shuffle_v4:
; GFX1250-SDAG: ; %bb.0: ; %bb
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -3850,6 +3996,8 @@ define amdgpu_kernel void @fadd_shuffle_v4(ptr addrspace(1) %arg) {
;
; GFX1250-GISEL-LABEL: fadd_shuffle_v4:
; GFX1250-GISEL: ; %bb.0: ; %bb
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v6, 0x3ff, v0
@@ -3912,6 +4060,8 @@ define amdgpu_kernel void @fneg_v2f32_vec(ptr addrspace(1) %a) {
;
; GFX1250-SDAG-LABEL: fneg_v2f32_vec:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -3925,6 +4075,8 @@ define amdgpu_kernel void @fneg_v2f32_vec(ptr addrspace(1) %a) {
;
; GFX1250-GISEL-LABEL: fneg_v2f32_vec:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_and_b32_e32 v2, 0x3ff, v0
@@ -3978,6 +4130,8 @@ define amdgpu_kernel void @fneg_v2f32_scalar(ptr addrspace(1) %a, <2 x float> %x
;
; GFX1250-SDAG-LABEL: fneg_v2f32_scalar:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -3990,6 +4144,8 @@ define amdgpu_kernel void @fneg_v2f32_scalar(ptr addrspace(1) %a, <2 x float> %x
;
; GFX1250-GISEL-LABEL: fneg_v2f32_scalar:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
index 4bb6979092be8..95fce2c9fa14a 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp64.ll
@@ -5,6 +5,8 @@
define amdgpu_kernel void @fadd_v2_vv(ptr addrspace(1) %a) {
; GFX1251-LABEL: fadd_v2_vv:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -25,6 +27,8 @@ define amdgpu_kernel void @fadd_v2_vv(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_vs(ptr addrspace(1) %a, <2 x double> %x) {
; GFX1251-LABEL: fadd_v2_vs:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -50,6 +54,8 @@ define amdgpu_kernel void @fadd_v2_vs(ptr addrspace(1) %a, <2 x double> %x) {
define amdgpu_kernel void @fadd_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x double> %y) {
; GFX1251-SDAG-LABEL: fadd_v2_ss:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x1
; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -67,6 +73,8 @@ define amdgpu_kernel void @fadd_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
;
; GFX1251-GISEL-LABEL: fadd_v2_ss:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x1
; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -97,6 +105,8 @@ define amdgpu_kernel void @fadd_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
; GFX1251-SDAG-LABEL: fadd_v4_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -123,6 +133,8 @@ define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
;
; GFX1251-GISEL-LABEL: fadd_v4_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -157,6 +169,8 @@ define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
; GFX1251-SDAG-LABEL: fadd_v32_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -272,6 +286,8 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
;
; GFX1251-GISEL-LABEL: fadd_v32_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -396,6 +412,8 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
define amdgpu_kernel void @fadd_v2_v_imm(ptr addrspace(1) %a) {
; GFX1251-LABEL: fadd_v2_v_imm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -416,6 +434,8 @@ define amdgpu_kernel void @fadd_v2_v_imm(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_v_v_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: fadd_v2_v_v_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -443,6 +463,8 @@ define amdgpu_kernel void @fadd_v2_v_v_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_v_lit_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: fadd_v2_v_lit_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -463,6 +485,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_v_lit_hi0(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fadd_v2_v_lit_hi0:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -477,6 +501,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_hi0(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fadd_v2_v_lit_hi0:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -501,6 +527,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_hi0(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_v_lit_lo0(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fadd_v2_v_lit_lo0:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -515,6 +543,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_lo0(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fadd_v2_v_lit_lo0:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -539,6 +569,8 @@ define amdgpu_kernel void @fadd_v2_v_lit_lo0(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fadd_v2_v_unfoldable_lit:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -553,6 +585,8 @@ define amdgpu_kernel void @fadd_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fadd_v2_v_unfoldable_lit:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -577,6 +611,8 @@ define amdgpu_kernel void @fadd_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-LABEL: fadd_v2_v_fneg:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -592,6 +628,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, double %x) {
;
; GFX1251-GISEL-LABEL: fadd_v2_v_fneg:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -623,6 +661,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, double %x) {
define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-LABEL: fadd_v2_v_fneg_lo:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -638,6 +678,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, double %x) {
;
; GFX1251-GISEL-LABEL: fadd_v2_v_fneg_lo:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -669,6 +711,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, double %x) {
define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-LABEL: fadd_v2_v_fneg_hi:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -684,6 +728,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, double %x) {
;
; GFX1251-GISEL-LABEL: fadd_v2_v_fneg_hi:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -715,6 +761,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, double %x) {
define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, double %x, double %y) {
; GFX1251-SDAG-LABEL: fadd_v2_v_fneg_lo2:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x1
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -733,6 +781,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, double %x, do
;
; GFX1251-GISEL-LABEL: fadd_v2_v_fneg_lo2:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x1
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -765,6 +815,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, double %x, do
define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, double %x, double %y) {
; GFX1251-SDAG-LABEL: fadd_v2_v_fneg_hi2:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x1
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -783,6 +835,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, double %x, do
;
; GFX1251-GISEL-LABEL: fadd_v2_v_fneg_hi2:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x1
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -815,6 +869,8 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, double %x, do
define amdgpu_kernel void @fmul_v2_vv(ptr addrspace(1) %a) {
; GFX1251-LABEL: fmul_v2_vv:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -835,6 +891,8 @@ define amdgpu_kernel void @fmul_v2_vv(ptr addrspace(1) %a) {
define amdgpu_kernel void @fmul_v2_vs(ptr addrspace(1) %a, <2 x double> %x) {
; GFX1251-LABEL: fmul_v2_vs:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -860,6 +918,8 @@ define amdgpu_kernel void @fmul_v2_vs(ptr addrspace(1) %a, <2 x double> %x) {
define amdgpu_kernel void @fmul_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x double> %y) {
; GFX1251-SDAG-LABEL: fmul_v2_ss:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x1
; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -877,6 +937,8 @@ define amdgpu_kernel void @fmul_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
;
; GFX1251-GISEL-LABEL: fmul_v2_ss:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x1
; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -907,6 +969,8 @@ define amdgpu_kernel void @fmul_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
; GFX1251-SDAG-LABEL: fmul_v4_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -933,6 +997,8 @@ define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
;
; GFX1251-GISEL-LABEL: fmul_v4_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -967,6 +1033,8 @@ define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
; GFX1251-SDAG-LABEL: fmul_v32_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1082,6 +1150,8 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
;
; GFX1251-GISEL-LABEL: fmul_v32_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1206,6 +1276,8 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
define amdgpu_kernel void @fmul_v2_v_imm(ptr addrspace(1) %a) {
; GFX1251-LABEL: fmul_v2_v_imm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1226,6 +1298,8 @@ define amdgpu_kernel void @fmul_v2_v_imm(ptr addrspace(1) %a) {
define amdgpu_kernel void @fmul_v2_v_v_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: fmul_v2_v_v_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1253,6 +1327,8 @@ define amdgpu_kernel void @fmul_v2_v_v_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @fmul_v2_v_lit_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: fmul_v2_v_lit_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1273,6 +1349,8 @@ define amdgpu_kernel void @fmul_v2_v_lit_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @fmul_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fmul_v2_v_unfoldable_lit:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1287,6 +1365,8 @@ define amdgpu_kernel void @fmul_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fmul_v2_v_unfoldable_lit:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1311,6 +1391,8 @@ define amdgpu_kernel void @fmul_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-LABEL: fmul_v2_v_fneg:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1326,6 +1408,8 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, double %x) {
;
; GFX1251-GISEL-LABEL: fmul_v2_v_fneg:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1357,6 +1441,8 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, double %x) {
define amdgpu_kernel void @fma_v2_vv(ptr addrspace(1) %a) {
; GFX1251-LABEL: fma_v2_vv:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1377,6 +1463,8 @@ define amdgpu_kernel void @fma_v2_vv(ptr addrspace(1) %a) {
define amdgpu_kernel void @fma_v2_vs(ptr addrspace(1) %a, <2 x double> %x) {
; GFX1251-LABEL: fma_v2_vs:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -1402,6 +1490,8 @@ define amdgpu_kernel void @fma_v2_vs(ptr addrspace(1) %a, <2 x double> %x) {
define amdgpu_kernel void @fma_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x double> %y, <2 x double> %z) {
; GFX1251-SDAG-LABEL: fma_v2_ss:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x2
; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -1422,6 +1512,8 @@ define amdgpu_kernel void @fma_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
;
; GFX1251-GISEL-LABEL: fma_v2_ss:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x2
; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -1455,6 +1547,8 @@ define amdgpu_kernel void @fma_v2_ss(ptr addrspace(1) %a, <2 x double> %x, <2 x
define amdgpu_kernel void @fma_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
; GFX1251-LABEL: fma_v4_vs:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1489,6 +1583,8 @@ define amdgpu_kernel void @fma_v4_vs(ptr addrspace(1) %a, <4 x double> %x) {
define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
; GFX1251-SDAG-LABEL: fma_v32_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1604,6 +1700,8 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
;
; GFX1251-GISEL-LABEL: fma_v32_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1728,6 +1826,8 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x double> %x) {
define amdgpu_kernel void @fma_v2_v_imm(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fma_v2_v_imm:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1743,6 +1843,8 @@ define amdgpu_kernel void @fma_v2_v_imm(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fma_v2_v_imm:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1768,6 +1870,8 @@ define amdgpu_kernel void @fma_v2_v_imm(ptr addrspace(1) %a) {
define amdgpu_kernel void @fma_v2_v_v_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: fma_v2_v_v_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1795,6 +1899,8 @@ define amdgpu_kernel void @fma_v2_v_v_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @fma_v2_v_lit_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: fma_v2_v_lit_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1815,6 +1921,8 @@ define amdgpu_kernel void @fma_v2_v_lit_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fma_v2_v_unfoldable_lit:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v12, 0x3ff, v0
@@ -1831,6 +1939,8 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fma_v2_v_unfoldable_lit:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[8:9], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v12, 0x3ff, v0
@@ -1860,6 +1970,8 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, double %x) {
; GFX1251-SDAG-LABEL: fma_v2_v_fneg:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1875,6 +1987,8 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, double %x) {
;
; GFX1251-GISEL-LABEL: fma_v2_v_fneg:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1906,6 +2020,8 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, double %x) {
define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) {
; GFX1251-SDAG-LABEL: fma_vector_vector_neg_scalar_lo_scalar_hi:
; GFX1251-SDAG: ; %bb.0: ; %bb
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -1924,6 +2040,8 @@ define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspa
;
; GFX1251-GISEL-LABEL: fma_vector_vector_neg_scalar_lo_scalar_hi:
; GFX1251-GISEL: ; %bb.0: ; %bb
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -2011,6 +2129,8 @@ bb:
define amdgpu_kernel void @fneg_v2f64_vec(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fneg_v2f64_vec:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -2023,6 +2143,8 @@ define amdgpu_kernel void @fneg_v2f64_vec(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fneg_v2f64_vec:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -2046,6 +2168,8 @@ define amdgpu_kernel void @fneg_v2f64_vec(ptr addrspace(1) %a) {
define amdgpu_kernel void @fneg_v2f64_scalar(ptr addrspace(1) %a, <2 x double> %x) {
; GFX1251-SDAG-LABEL: fneg_v2f64_scalar:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x1
; GFX1251-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
@@ -2062,6 +2186,8 @@ define amdgpu_kernel void @fneg_v2f64_scalar(ptr addrspace(1) %a, <2 x double> %
;
; GFX1251-GISEL-LABEL: fneg_v2f64_scalar:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x1
; GFX1251-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
@@ -2103,6 +2229,8 @@ define amdgpu_kernel void @fneg_v2f64_scalar(ptr addrspace(1) %a, <2 x double> %
define amdgpu_kernel void @fneg_v32f64_vec(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: fneg_v32f64_vec:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2176,6 +2304,8 @@ define amdgpu_kernel void @fneg_v32f64_vec(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: fneg_v32f64_vec:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -2294,6 +2424,8 @@ define amdgpu_kernel void @fneg_v32f64_vec(ptr addrspace(1) %a) {
define amdgpu_kernel void @fneg_v2f64_pkfma(ptr addrspace(1) %out) {
; GFX1251-SDAG-LABEL: fneg_v2f64_pkfma:
; GFX1251-SDAG: ; %bb.0: ; %entry
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
@@ -2310,6 +2442,8 @@ define amdgpu_kernel void @fneg_v2f64_pkfma(ptr addrspace(1) %out) {
;
; GFX1251-GISEL-LABEL: fneg_v2f64_pkfma:
; GFX1251-GISEL: ; %bb.0: ; %entry
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
diff --git a/llvm/test/CodeGen/AMDGPU/packed-u64.ll b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
index 13d6b0b9e93ce..7ee1a55a6be03 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-u64.ll
@@ -5,6 +5,8 @@
define amdgpu_kernel void @add_v2_vv(ptr addrspace(1) %a) {
; GFX1251-LABEL: add_v2_vv:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -25,6 +27,8 @@ define amdgpu_kernel void @add_v2_vv(ptr addrspace(1) %a) {
define amdgpu_kernel void @add_v2_vs(ptr addrspace(1) %a, <2 x i64> %x) {
; GFX1251-LABEL: add_v2_vs:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -50,6 +54,8 @@ define amdgpu_kernel void @add_v2_vs(ptr addrspace(1) %a, <2 x i64> %x) {
define amdgpu_kernel void @add_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64> %y) {
; GFX1251-SDAG-LABEL: add_v2_ss:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x1
; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -67,6 +73,8 @@ define amdgpu_kernel void @add_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
;
; GFX1251-GISEL-LABEL: add_v2_ss:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x1
; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -97,6 +105,8 @@ define amdgpu_kernel void @add_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
define amdgpu_kernel void @add_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
; GFX1251-SDAG-LABEL: add_v4_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -123,6 +133,8 @@ define amdgpu_kernel void @add_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
;
; GFX1251-GISEL-LABEL: add_v4_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -157,6 +169,8 @@ define amdgpu_kernel void @add_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
define amdgpu_kernel void @add_v32_vs(ptr addrspace(1) %a, <32 x i64> %x) {
; GFX1251-SDAG-LABEL: add_v32_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -272,6 +286,8 @@ define amdgpu_kernel void @add_v32_vs(ptr addrspace(1) %a, <32 x i64> %x) {
;
; GFX1251-GISEL-LABEL: add_v32_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -396,6 +412,8 @@ define amdgpu_kernel void @add_v32_vs(ptr addrspace(1) %a, <32 x i64> %x) {
define amdgpu_kernel void @add_v2_v_imm(ptr addrspace(1) %a) {
; GFX1251-LABEL: add_v2_v_imm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -416,6 +434,8 @@ define amdgpu_kernel void @add_v2_v_imm(ptr addrspace(1) %a) {
define amdgpu_kernel void @add_v2_v_v_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: add_v2_v_v_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -443,6 +463,8 @@ define amdgpu_kernel void @add_v2_v_v_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @add_v2_v_lit_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: add_v2_v_lit_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -463,6 +485,8 @@ define amdgpu_kernel void @add_v2_v_lit_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @add_v2_v_lit_hi0(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: add_v2_v_lit_hi0:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -477,6 +501,8 @@ define amdgpu_kernel void @add_v2_v_lit_hi0(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: add_v2_v_lit_hi0:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -501,6 +527,8 @@ define amdgpu_kernel void @add_v2_v_lit_hi0(ptr addrspace(1) %a) {
define amdgpu_kernel void @add_v2_v_lit_lo0(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: add_v2_v_lit_lo0:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -515,6 +543,8 @@ define amdgpu_kernel void @add_v2_v_lit_lo0(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: add_v2_v_lit_lo0:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -539,6 +569,8 @@ define amdgpu_kernel void @add_v2_v_lit_lo0(ptr addrspace(1) %a) {
define amdgpu_kernel void @add_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: add_v2_v_unfoldable_lit:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -553,6 +585,8 @@ define amdgpu_kernel void @add_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: add_v2_v_unfoldable_lit:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -577,6 +611,8 @@ define amdgpu_kernel void @add_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
define amdgpu_kernel void @sub_v2_vv(ptr addrspace(1) %a, ptr addrspace(1) %b) {
; GFX1251-LABEL: sub_v2_vv:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -601,6 +637,8 @@ define amdgpu_kernel void @sub_v2_vv(ptr addrspace(1) %a, ptr addrspace(1) %b) {
define amdgpu_kernel void @sub_v2_vs(ptr addrspace(1) %a, <2 x i64> %x) {
; GFX1251-LABEL: sub_v2_vs:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
@@ -626,6 +664,8 @@ define amdgpu_kernel void @sub_v2_vs(ptr addrspace(1) %a, <2 x i64> %x) {
define amdgpu_kernel void @sub_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64> %y) {
; GFX1251-SDAG-LABEL: sub_v2_ss:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_clause 0x1
; GFX1251-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -643,6 +683,8 @@ define amdgpu_kernel void @sub_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
;
; GFX1251-GISEL-LABEL: sub_v2_ss:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_clause 0x1
; GFX1251-GISEL-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -673,6 +715,8 @@ define amdgpu_kernel void @sub_v2_ss(ptr addrspace(1) %a, <2 x i64> %x, <2 x i64
define amdgpu_kernel void @sub_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
; GFX1251-SDAG-LABEL: sub_v4_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -699,6 +743,8 @@ define amdgpu_kernel void @sub_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
;
; GFX1251-GISEL-LABEL: sub_v4_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -733,6 +779,8 @@ define amdgpu_kernel void @sub_v4_vs(ptr addrspace(1) %a, <4 x i64> %x) {
define amdgpu_kernel void @sub_v32_vs(ptr addrspace(1) %a, <32 x i64> %x) {
; GFX1251-SDAG-LABEL: sub_v32_vs:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -848,6 +896,8 @@ define amdgpu_kernel void @sub_v32_vs(ptr addrspace(1) %a, <32 x i64> %x) {
;
; GFX1251-GISEL-LABEL: sub_v32_vs:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -972,6 +1022,8 @@ define amdgpu_kernel void @sub_v32_vs(ptr addrspace(1) %a, <32 x i64> %x) {
define amdgpu_kernel void @sub_v2_v_imm(ptr addrspace(1) %a) {
; GFX1251-LABEL: sub_v2_v_imm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -992,6 +1044,8 @@ define amdgpu_kernel void @sub_v2_v_imm(ptr addrspace(1) %a) {
define amdgpu_kernel void @sub_v2_imm_v(ptr addrspace(1) %a) {
; GFX1251-LABEL: sub_v2_imm_v:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1012,6 +1066,8 @@ define amdgpu_kernel void @sub_v2_imm_v(ptr addrspace(1) %a) {
define amdgpu_kernel void @sub_v2_v_v_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: sub_v2_v_v_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -1039,6 +1095,8 @@ define amdgpu_kernel void @sub_v2_v_v_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @sub_v2_v_lit_splat(ptr addrspace(1) %a) {
; GFX1251-LABEL: sub_v2_v_lit_splat:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -1059,6 +1117,8 @@ define amdgpu_kernel void @sub_v2_v_lit_splat(ptr addrspace(1) %a) {
define amdgpu_kernel void @sub_v2_v_lit_hi0(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: sub_v2_v_lit_hi0:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1073,6 +1133,8 @@ define amdgpu_kernel void @sub_v2_v_lit_hi0(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: sub_v2_v_lit_hi0:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1097,6 +1159,8 @@ define amdgpu_kernel void @sub_v2_v_lit_hi0(ptr addrspace(1) %a) {
define amdgpu_kernel void @sub_v2_v_lit_lo0(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: sub_v2_v_lit_lo0:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1111,6 +1175,8 @@ define amdgpu_kernel void @sub_v2_v_lit_lo0(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: sub_v2_v_lit_lo0:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1135,6 +1201,8 @@ define amdgpu_kernel void @sub_v2_v_lit_lo0(ptr addrspace(1) %a) {
define amdgpu_kernel void @sub_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
; GFX1251-SDAG-LABEL: sub_v2_v_unfoldable_lit:
; GFX1251-SDAG: ; %bb.0:
+; GFX1251-SDAG-NEXT: global_wb
+; GFX1251-SDAG-NEXT: v_nop
; GFX1251-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1251-SDAG-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -1149,6 +1217,8 @@ define amdgpu_kernel void @sub_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
;
; GFX1251-GISEL-LABEL: sub_v2_v_unfoldable_lit:
; GFX1251-GISEL: ; %bb.0:
+; GFX1251-GISEL-NEXT: global_wb
+; GFX1251-GISEL-NEXT: v_nop
; GFX1251-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
; GFX1251-GISEL-NEXT: v_and_b32_e32 v8, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
index c4d2541e7fcf8..34e4202eda23f 100644
--- a/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll
@@ -49,6 +49,8 @@ define <2 x i64> @pk_lshl_add_u64_vvv(<2 x i64> %v, <2 x i64> %s, <2 x i64> %a)
define amdgpu_kernel void @pk_lshl_add_u64_s2v(<2 x i64> %v) {
; GFX1251-LABEL: pk_lshl_add_u64_s2v:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: flat_load_b128 v[0:3], v[0:1]
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -73,6 +75,8 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2v(<2 x i64> %v) {
define amdgpu_kernel void @pk_lshl_add_u64_v2s(<2 x i64> %a) {
; GFX1251-LABEL: pk_lshl_add_u64_v2s:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: flat_load_b128 v[0:3], v[0:1]
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
@@ -97,6 +101,8 @@ define amdgpu_kernel void @pk_lshl_add_u64_v2s(<2 x i64> %a) {
define amdgpu_kernel void @pk_lshl_add_u64_s2s(<2 x i64> %v, <2 x i64> %a) {
; GFX1251-LABEL: pk_lshl_add_u64_s2s:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1251-NEXT: s_wait_kmcnt 0x0
@@ -172,6 +178,8 @@ define i32 @pk_lshl_add_u64_maybe_oob(<2 x ptr> %p, <2 x i32> %i) {
define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_3(<2 x i64> %v, <2 x i64> %a) {
; GFX1251-LABEL: pk_lshl_add_u64_s2s_shift2_3:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1251-NEXT: s_wait_kmcnt 0x0
@@ -196,6 +204,8 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_3(<2 x i64> %v, <2 x i64>
define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_4(<2 x i64> %v, <2 x i64> %a) {
; GFX1251-LABEL: pk_lshl_add_u64_s2s_shift2_4:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1251-NEXT: s_wait_kmcnt 0x0
@@ -218,6 +228,8 @@ define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_4(<2 x i64> %v, <2 x i64>
define amdgpu_kernel void @pk_lshl_add_u64_s2s_shift2_5(<2 x i64> %v, <2 x i64> %a) {
; GFX1251-LABEL: pk_lshl_add_u64_s2s_shift2_5:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GFX1251-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index 344d1f5d0b854..a8db3da8b3216 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -35,6 +35,8 @@ define amdgpu_kernel void @preload_block_count_x(ptr addrspace(1) inreg %out) #0
;
; GFX1250-LABEL: preload_block_count_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -77,6 +79,8 @@ define amdgpu_kernel void @preload_unused_arg_block_count_x(ptr addrspace(1) inr
;
; GFX1250-LABEL: preload_unused_arg_block_count_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -120,6 +124,8 @@ define amdgpu_kernel void @no_free_sgprs_block_count_x(ptr addrspace(1) inreg %o
;
; GFX1250-LABEL: no_free_sgprs_block_count_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
; GFX1250-NEXT: global_store_b32 v0, v1, s[8:9]
@@ -153,6 +159,8 @@ define amdgpu_kernel void @no_inreg_block_count_x(ptr addrspace(1) %out) #0 {
;
; GFX1250-LABEL: no_inreg_block_count_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[0:1], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -191,6 +199,8 @@ define amdgpu_kernel void @mixed_inreg_block_count_x(ptr addrspace(1) %out, i32
;
; GFX1250-LABEL: mixed_inreg_block_count_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b32 s4, s[0:1], 0x10 nv
@@ -238,6 +248,8 @@ define amdgpu_kernel void @incorrect_type_i64_block_count_x(ptr addrspace(1) inr
;
; GFX1250-LABEL: incorrect_type_i64_block_count_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[0:1], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -284,6 +296,8 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
;
; GFX1250-FAKE16-LABEL: incorrect_type_i16_block_count_x:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-FAKE16-NEXT: global_load_u16 v1, v0, s[0:1] offset:8 nv
@@ -293,6 +307,8 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
;
; GFX1250-REAL16-LABEL: incorrect_type_i16_block_count_x:
; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-REAL16-NEXT: global_load_u16 v0, v1, s[0:1] offset:8 nv
@@ -335,6 +351,8 @@ define amdgpu_kernel void @preload_block_count_y(ptr addrspace(1) inreg %out) #0
;
; GFX1250-LABEL: preload_block_count_y:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -379,6 +397,8 @@ define amdgpu_kernel void @random_incorrect_offset(ptr addrspace(1) inreg %out)
;
; GFX1250-LABEL: random_incorrect_offset:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b32 s0, s[0:1], 0xa nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -424,6 +444,8 @@ define amdgpu_kernel void @preload_block_count_z(ptr addrspace(1) inreg %out) #0
;
; GFX1250-LABEL: preload_block_count_z:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -471,6 +493,8 @@ define amdgpu_kernel void @preload_block_count_x_imparg_align_ptr_i8(ptr addrspa
;
; GFX1250-LABEL: preload_block_count_x_imparg_align_ptr_i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s4, 0xff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -522,6 +546,8 @@ define amdgpu_kernel void @preload_block_count_xyz(ptr addrspace(1) inreg %out)
;
; GFX1250-LABEL: preload_block_count_xyz:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
; GFX1250-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6
@@ -574,6 +600,8 @@ define amdgpu_kernel void @preload_workgroup_size_x(ptr addrspace(1) inreg %out)
;
; GFX1250-LABEL: preload_workgroup_size_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s7, 0xffff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -621,6 +649,8 @@ define amdgpu_kernel void @preload_workgroup_size_y(ptr addrspace(1) inreg %out)
;
; GFX1250-LABEL: preload_workgroup_size_y:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s7, 16
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -670,6 +700,8 @@ define amdgpu_kernel void @preload_workgroup_size_z(ptr addrspace(1) inreg %out)
;
; GFX1250-LABEL: preload_workgroup_size_z:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s8, 0xffff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -727,6 +759,8 @@ define amdgpu_kernel void @preload_workgroup_size_xyz(ptr addrspace(1) inreg %ou
;
; GFX1250-LABEL: preload_workgroup_size_xyz:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s7, 16
; GFX1250-NEXT: s_and_b32 s1, s7, 0xffff
@@ -787,6 +821,8 @@ define amdgpu_kernel void @preload_remainder_x(ptr addrspace(1) inreg %out) #0 {
;
; GFX1250-LABEL: preload_remainder_x:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s8, 16
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -834,6 +870,8 @@ define amdgpu_kernel void @preloadremainder_y(ptr addrspace(1) inreg %out) #0 {
;
; GFX1250-LABEL: preloadremainder_y:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s9, 0xffff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -881,6 +919,8 @@ define amdgpu_kernel void @preloadremainder_z(ptr addrspace(1) inreg %out) #0 {
;
; GFX1250-LABEL: preloadremainder_z:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s9, 16
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -936,6 +976,8 @@ define amdgpu_kernel void @preloadremainder_xyz(ptr addrspace(1) inreg %out) #0
;
; GFX1250-LABEL: preloadremainder_xyz:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s9, 16
; GFX1250-NEXT: s_lshr_b32 s1, s8, 16
@@ -994,6 +1036,8 @@ define amdgpu_kernel void @no_free_sgprs_preloadremainder_z(ptr addrspace(1) inr
;
; GFX1250-LABEL: no_free_sgprs_preloadremainder_z:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s15, 16
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1043,6 +1087,8 @@ define amdgpu_kernel void @preload_block_max_user_sgprs(ptr addrspace(1) inreg %
;
; GFX1250-LABEL: preload_block_max_user_sgprs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s12
; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -1092,6 +1138,8 @@ define amdgpu_kernel void @preload_block_count_z_workgroup_size_z_remainder_z(pt
;
; GFX1250-LABEL: preload_block_count_z_workgroup_size_z_remainder_z:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s9, 16
; GFX1250-NEXT: s_and_b32 s1, s8, 0xffff
diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
index 70d08930e3b9c..3cd240dce4a69 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
@@ -36,6 +36,8 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) inreg %out, i8 inreg %arg0)
;
; GFX1250-LABEL: ptr1_i8:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s4, 0xff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -80,6 +82,8 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) inreg %out, i8 zero
;
; GFX1250-LABEL: ptr1_i8_zext_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s4, 0xff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -124,6 +128,8 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) inreg %out, i16
;
; GFX1250-LABEL: ptr1_i16_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s4, 0xffff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -166,6 +172,8 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) inreg %out, i32
;
; GFX1250-LABEL: ptr1_i32_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -209,6 +217,8 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 inreg %arg0, ptr addrspa
;
; GFX1250-LABEL: i32_ptr1_i32_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_co_i32 s0, s2, s6
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -257,6 +267,8 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) inreg %out,
;
; GFX1250-LABEL: ptr1_i16_i16_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_lshr_b32 s0, s4, 16
; GFX1250-NEXT: s_and_b32 s1, s4, 0xffff
@@ -303,6 +315,8 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) inreg %out, <2
;
; GFX1250-LABEL: ptr1_v2i8_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: global_store_b16 v0, v1, s[2:3]
@@ -353,6 +367,8 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) inreg %out, ptr ad
;
; GFX1250-LABEL: byref_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[4:5], s[0:1], 0x100 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -413,6 +429,8 @@ define amdgpu_kernel void @byref_staggered_preload_arg(ptr addrspace(1) inreg %o
;
; GFX1250-LABEL: byref_staggered_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[4:5], s[0:1], 0x100 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -482,6 +500,8 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture inreg %out, <8 x
;
; GFX1250-LABEL: v8i32_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v0, s14
; GFX1250-NEXT: v_dual_mov_b32 v1, s15 :: v_dual_mov_b32 v2, s16
@@ -530,6 +550,8 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture inreg %o
;
; GFX1250-LABEL: v3i16_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
; GFX1250-NEXT: v_mov_b32_e32 v2, s4
@@ -576,6 +598,8 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture inreg %o
;
; GFX1250-LABEL: v3i32_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1250-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, 0
@@ -620,6 +644,8 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture inreg %o
;
; GFX1250-LABEL: v3f32_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
; GFX1250-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s8
@@ -677,6 +703,8 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture inreg %ou
;
; GFX1250-LABEL: v5i8_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_pack_lh_b32_b16 s0, 0, s4
; GFX1250-NEXT: s_and_b32 s1, s4, 0xffff
@@ -749,6 +777,8 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture inreg %out, <5 x
;
; GFX1250-LABEL: v5f64_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b64 s[12:13], s[0:1], 0x60 nv
@@ -829,6 +859,8 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) inreg %out, <8 x i8
;
; GFX1250-LABEL: v8i8_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_pack_lh_b32_b16 s0, 0, s5
; GFX1250-NEXT: s_pack_lh_b32_b16 s1, 0, s4
@@ -875,6 +907,8 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) inreg %out, i
;
; GFX1250-LABEL: i64_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -914,6 +948,8 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) inreg %out, d
;
; GFX1250-LABEL: f64_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
@@ -954,6 +990,8 @@ define amdgpu_kernel void @half_kernel_preload_arg(ptr addrspace(1) inreg %out,
;
; GFX1250-LABEL: half_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: global_store_b16 v0, v1, s[2:3]
@@ -993,6 +1031,8 @@ define amdgpu_kernel void @bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out
;
; GFX1250-LABEL: bfloat_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: global_store_b16 v0, v1, s[2:3]
@@ -1032,6 +1072,8 @@ define amdgpu_kernel void @v2bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
;
; GFX1250-LABEL: v2bfloat_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: global_store_b32 v0, v1, s[2:3]
@@ -1074,6 +1116,8 @@ define amdgpu_kernel void @v3bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
;
; GFX1250-LABEL: v3bfloat_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
; GFX1250-NEXT: v_mov_b32_e32 v2, s4
@@ -1120,6 +1164,8 @@ define amdgpu_kernel void @v6bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
;
; GFX1250-LABEL: v6bfloat_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1250-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, 0
@@ -1173,6 +1219,8 @@ define amdgpu_kernel void @half_v7bfloat_kernel_preload_arg(ptr addrspace(1) inr
;
; GFX1250-LABEL: half_v7bfloat_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, s4
; GFX1250-NEXT: v_dual_mov_b32 v5, s9 :: v_dual_mov_b32 v2, s8
@@ -1220,6 +1268,8 @@ define amdgpu_kernel void @i1_kernel_preload_arg(ptr addrspace(1) inreg %out, i1
;
; GFX1250-LABEL: i1_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s4, 1
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1267,6 +1317,8 @@ define amdgpu_kernel void @fp128_kernel_preload_arg(ptr addrspace(1) inreg %out,
;
; GFX1250-LABEL: fp128_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1250-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, s9
@@ -1327,6 +1379,8 @@ define amdgpu_kernel void @v7i8_kernel_preload_arg(ptr addrspace(1) inreg %out,
;
; GFX1250-LABEL: v7i8_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_pack_lh_b32_b16 s0, 0, s4
; GFX1250-NEXT: s_and_b32 s1, s4, 0xffff
@@ -1382,6 +1436,8 @@ define amdgpu_kernel void @v7half_kernel_preload_arg(ptr addrspace(1) inreg %out
;
; GFX1250-LABEL: v7half_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, s9
; GFX1250-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v0, s6
@@ -1429,6 +1485,8 @@ define amdgpu_kernel void @i16_i32_kernel_preload_arg(ptr addrspace(1) inreg %ou
;
; GFX1250-LABEL: i16_i32_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: v_mov_b32_e32 v2, s5
@@ -1481,6 +1539,8 @@ define amdgpu_kernel void @i16_v3i32_kernel_preload_arg(ptr addrspace(1) inreg %
;
; GFX1250-LABEL: i16_v3i32_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, s4
; GFX1250-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
@@ -1527,6 +1587,8 @@ define amdgpu_kernel void @i16_i16_kernel_preload_arg(ptr addrspace(1) inreg %ou
;
; GFX1250-LABEL: i16_i16_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: s_clause 0x1
@@ -1581,6 +1643,8 @@ define amdgpu_kernel void @i16_v2i8_kernel_preload_arg(ptr addrspace(1) inreg %o
;
; GFX1250-LABEL: i16_v2i8_kernel_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
; GFX1250-NEXT: s_clause 0x1
@@ -1631,6 +1695,8 @@ define amdgpu_kernel void @i32_ptr1_i32_staggered_preload_arg(i32 inreg %arg0, p
;
; GFX1250-LABEL: i32_ptr1_i32_staggered_preload_arg:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[4:6], s[0:1], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1676,6 +1742,8 @@ define amdgpu_kernel void @ptr1_i8_trailing_unused(ptr addrspace(1) inreg %out,
;
; GFX1250-LABEL: ptr1_i8_trailing_unused:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s4, 0xff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
index a43dda6da063e..85ccdf81bafa7 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -10,6 +10,8 @@ declare i32 @llvm.amdgcn.workitem.id.x()
define amdgpu_kernel void @promote_async_load_offset_negative(ptr addrspace(1) %src) {
; GFX1250-LABEL: promote_async_load_offset_negative:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -57,6 +59,8 @@ entry:
define amdgpu_kernel void @promote_async_load_offset_positive(ptr addrspace(1) %src) {
; GFX1250-LABEL: promote_async_load_offset_positive:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
@@ -103,6 +107,8 @@ entry:
define amdgpu_kernel void @promote_async_store_offset_negative(ptr addrspace(1) %dst) {
; GFX1250-LABEL: promote_async_store_offset_negative:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
index 5f534f58f7342..13d43c2d00835 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -5289,6 +5289,8 @@ define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) %
;
; GFX1250-LABEL: compute_mad:
; GFX1250: ; %bb.0: ; %bb
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -5442,6 +5444,8 @@ define amdgpu_ps i32 @s_mul_add_1_i32(i32 inreg %x, i32 inreg %y) {
;
; GFX1250-LABEL: s_mul_add_1_i32:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_co_i32 s1, s1, 1
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -5486,6 +5490,8 @@ define amdgpu_ps i32 @s_mul_add_1_i32_commute(i32 inreg %x, i32 inreg %y) {
;
; GFX1250-LABEL: s_mul_add_1_i32_commute:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_add_co_i32 s1, s1, 1
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/s-barrier-signal-var-gep.ll b/llvm/test/CodeGen/AMDGPU/s-barrier-signal-var-gep.ll
index e8b7f32c5fc89..15194df6aa09e 100644
--- a/llvm/test/CodeGen/AMDGPU/s-barrier-signal-var-gep.ll
+++ b/llvm/test/CodeGen/AMDGPU/s-barrier-signal-var-gep.ll
@@ -15,6 +15,8 @@
define amdgpu_kernel void @signal_var_bar0() {
; CHECK-LABEL: signal_var_bar0:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_mov_b32 m0, 0x100001
; CHECK-NEXT: s_barrier_init m0
@@ -24,6 +26,8 @@ define amdgpu_kernel void @signal_var_bar0() {
;
; CHECK-OBJ-SDAG-LABEL: signal_var_bar0:
; CHECK-OBJ-SDAG: ; %bb.0:
+; CHECK-OBJ-SDAG-NEXT: global_wb
+; CHECK-OBJ-SDAG-NEXT: v_nop
; CHECK-OBJ-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-SDAG-NEXT: s_mov_b32 s0, __amdgpu_named_barrier.bars.5a19a560517f8a3a4347b4502da34a70 at abs32@lo
; CHECK-OBJ-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -37,6 +41,8 @@ define amdgpu_kernel void @signal_var_bar0() {
;
; CHECK-OBJ-GISEL-LABEL: signal_var_bar0:
; CHECK-OBJ-GISEL: ; %bb.0:
+; CHECK-OBJ-GISEL-NEXT: global_wb
+; CHECK-OBJ-GISEL-NEXT: v_nop
; CHECK-OBJ-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-GISEL-NEXT: s_lshr_b32 s0, __amdgpu_named_barrier.bars.5a19a560517f8a3a4347b4502da34a70 at abs32@lo, 4
; CHECK-OBJ-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -56,6 +62,8 @@ define amdgpu_kernel void @signal_var_bar0() {
define amdgpu_kernel void @signal_var_bar1() {
; CHECK-LABEL: signal_var_bar1:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_mov_b32 m0, 0x100002
; CHECK-NEXT: s_barrier_init m0
@@ -65,6 +73,8 @@ define amdgpu_kernel void @signal_var_bar1() {
;
; CHECK-OBJ-SDAG-LABEL: signal_var_bar1:
; CHECK-OBJ-SDAG: ; %bb.0:
+; CHECK-OBJ-SDAG-NEXT: global_wb
+; CHECK-OBJ-SDAG-NEXT: v_nop
; CHECK-OBJ-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-SDAG-NEXT: s_mov_b32 s0, __amdgpu_named_barrier.bars.5a19a560517f8a3a4347b4502da34a70 at abs32@lo+16
; CHECK-OBJ-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -78,6 +88,8 @@ define amdgpu_kernel void @signal_var_bar1() {
;
; CHECK-OBJ-GISEL-LABEL: signal_var_bar1:
; CHECK-OBJ-GISEL: ; %bb.0:
+; CHECK-OBJ-GISEL-NEXT: global_wb
+; CHECK-OBJ-GISEL-NEXT: v_nop
; CHECK-OBJ-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-GISEL-NEXT: s_add_co_u32 s0, __amdgpu_named_barrier.bars.5a19a560517f8a3a4347b4502da34a70 at abs32@lo, 16
; CHECK-OBJ-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -104,6 +116,8 @@ define amdgpu_kernel void @signal_var_bar1() {
define amdgpu_kernel void @signal_var_misaligned() {
; CHECK-LABEL: signal_var_misaligned:
; CHECK: ; %bb.0:
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_mov_b32 m0, 0x100001
; CHECK-NEXT: s_barrier_init m0
@@ -113,6 +127,8 @@ define amdgpu_kernel void @signal_var_misaligned() {
;
; CHECK-OBJ-SDAG-LABEL: signal_var_misaligned:
; CHECK-OBJ-SDAG: ; %bb.0:
+; CHECK-OBJ-SDAG-NEXT: global_wb
+; CHECK-OBJ-SDAG-NEXT: v_nop
; CHECK-OBJ-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-SDAG-NEXT: s_mov_b32 s0, __amdgpu_named_barrier.bars.5a19a560517f8a3a4347b4502da34a70 at abs32@lo+1
; CHECK-OBJ-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -126,6 +142,8 @@ define amdgpu_kernel void @signal_var_misaligned() {
;
; CHECK-OBJ-GISEL-LABEL: signal_var_misaligned:
; CHECK-OBJ-GISEL: ; %bb.0:
+; CHECK-OBJ-GISEL-NEXT: global_wb
+; CHECK-OBJ-GISEL-NEXT: v_nop
; CHECK-OBJ-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-GISEL-NEXT: s_add_co_u32 s0, __amdgpu_named_barrier.bars.5a19a560517f8a3a4347b4502da34a70 at abs32@lo, 1
; CHECK-OBJ-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -152,6 +170,8 @@ define amdgpu_kernel void @signal_var_misaligned() {
define amdgpu_kernel void @signal_var_dynamic(i32 %idx) {
; CHECK-SDAG-LABEL: signal_var_dynamic:
; CHECK-SDAG: ; %bb.0:
+; CHECK-SDAG-NEXT: global_wb
+; CHECK-SDAG-NEXT: v_nop
; CHECK-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; CHECK-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -167,6 +187,8 @@ define amdgpu_kernel void @signal_var_dynamic(i32 %idx) {
;
; CHECK-GISEL-LABEL: signal_var_dynamic:
; CHECK-GISEL: ; %bb.0:
+; CHECK-GISEL-NEXT: global_wb
+; CHECK-GISEL-NEXT: v_nop
; CHECK-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; CHECK-GISEL-NEXT: s_wait_kmcnt 0x0
@@ -185,6 +207,8 @@ define amdgpu_kernel void @signal_var_dynamic(i32 %idx) {
;
; CHECK-OBJ-SDAG-LABEL: signal_var_dynamic:
; CHECK-OBJ-SDAG: ; %bb.0:
+; CHECK-OBJ-SDAG-NEXT: global_wb
+; CHECK-OBJ-SDAG-NEXT: v_nop
; CHECK-OBJ-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; CHECK-OBJ-SDAG-NEXT: s_wait_kmcnt 0x0
@@ -200,6 +224,8 @@ define amdgpu_kernel void @signal_var_dynamic(i32 %idx) {
;
; CHECK-OBJ-GISEL-LABEL: signal_var_dynamic:
; CHECK-OBJ-GISEL: ; %bb.0:
+; CHECK-OBJ-GISEL-NEXT: global_wb
+; CHECK-OBJ-GISEL-NEXT: v_nop
; CHECK-OBJ-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-OBJ-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; CHECK-OBJ-GISEL-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/s-cluster-barrier.ll b/llvm/test/CodeGen/AMDGPU/s-cluster-barrier.ll
index e217c1a629ca2..aa59380179471 100644
--- a/llvm/test/CodeGen/AMDGPU/s-cluster-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/s-cluster-barrier.ll
@@ -5,6 +5,8 @@
define amdgpu_kernel void @kernel1() #0 {
; GFX12-LABEL: kernel1:
; GFX12: ; %bb.0:
+; GFX12-NEXT: global_wb
+; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_cmp_eq_u32 0, 0
; GFX12-NEXT: s_barrier_signal_isfirst -1
diff --git a/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll b/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll
index 0b907ebfc1ed1..e7c299f04dc0d 100644
--- a/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll
@@ -7,6 +7,8 @@
define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in) #0 {
; GFX1250-SDAG-LABEL: kernel1:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GFX1250-SDAG-NEXT: s_mov_b32 m0, 1
@@ -18,6 +20,8 @@ define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in)
;
; GFX1250-GISEL-LABEL: kernel1:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
; GFX1250-GISEL-NEXT: s_wakeup_barrier 1
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
index 95d48915e6ac9..5a90551c16225 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll
@@ -7,6 +7,8 @@
define amdgpu_ps float @flat_load_b32_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b32_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v0, s[0:1] scale_offset
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -21,6 +23,8 @@ entry:
define amdgpu_ps float @flat_load_b32_idx32(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b32_idx32:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v0, s[0:1] scale_offset
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -34,6 +38,8 @@ entry:
define amdgpu_ps float @flat_load_b32_idxprom_wrong_stride(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b32_idxprom_wrong_stride:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -51,6 +57,8 @@ entry:
define amdgpu_ps float @flat_load_b16_idxprom_ioffset(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b16_idxprom_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_u16 v0, v0, s[0:1] offset:32 scale_offset
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -68,6 +76,8 @@ entry:
define amdgpu_ps <2 x float> @flat_load_b64_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b64_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b64 v[0:1], v0, s[0:1] scale_offset
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -82,6 +92,8 @@ entry:
define amdgpu_ps <3 x float> @flat_load_b96_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b96_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b96 v[0:2], v0, s[0:1] scale_offset
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -96,6 +108,8 @@ entry:
define amdgpu_ps <3 x float> @flat_load_b96_idxpromi_ioffset(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b96_idxpromi_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b96 v[0:2], v0, s[0:1] offset:192 scale_offset
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -111,6 +125,8 @@ entry:
define amdgpu_ps <4 x float> @flat_load_b128_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_load_b128_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b128 v[0:3], v0, s[0:1] scale_offset
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -125,6 +141,8 @@ entry:
define amdgpu_ps float @flat_load_b32_idxprom_range(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b32_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -142,6 +160,8 @@ entry:
define amdgpu_ps float @flat_load_b32_idxprom_range_ioffset(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b32_idxprom_range_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -162,6 +182,8 @@ entry:
define amdgpu_ps float @flat_load_b8_idxprom_range_ioffset(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b8_idxprom_range_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -182,6 +204,8 @@ entry:
define amdgpu_ps float @flat_load_b16_idxprom_range(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b16_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -201,6 +225,8 @@ entry:
define amdgpu_ps float @flat_load_b16_idxprom_range_ioffset(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b16_idxprom_range_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -221,6 +247,8 @@ entry:
define amdgpu_ps <2 x float> @flat_load_b64_idxprom_range(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b64_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -238,6 +266,8 @@ entry:
define amdgpu_ps <3 x float> @flat_load_b96_idxprom_range(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b96_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -255,6 +285,8 @@ entry:
define amdgpu_ps <3 x float> @flat_load_b96_idxprom_range_ioffset(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b96_idxprom_range_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -273,6 +305,8 @@ entry:
define amdgpu_ps <4 x float> @flat_load_b128_idxprom_range(ptr align 4 inreg %p, ptr align 4 %pp) {
; GCN-LABEL: flat_load_b128_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: flat_load_b32 v0, v[0:1]
; GCN-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -290,6 +324,8 @@ entry:
define amdgpu_ps void @flat_store_b32_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_store_b32_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v1, 1.0
; GCN-NEXT: flat_store_b32 v0, v1, s[0:1] scale_offset
@@ -304,6 +340,8 @@ entry:
define amdgpu_ps void @flat_store_b16_idxprom(ptr align 2 inreg %p, i32 %idx) {
; GCN-FAKE16-LABEL: flat_store_b16_idxprom:
; GCN-FAKE16: ; %bb.0: ; %entry
+; GCN-FAKE16-NEXT: global_wb
+; GCN-FAKE16-NEXT: v_nop
; GCN-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-FAKE16-NEXT: v_mov_b32_e32 v1, 1
; GCN-FAKE16-NEXT: flat_store_b16 v0, v1, s[0:1] scale_offset
@@ -311,6 +349,8 @@ define amdgpu_ps void @flat_store_b16_idxprom(ptr align 2 inreg %p, i32 %idx) {
;
; GCN-REAL16-LABEL: flat_store_b16_idxprom:
; GCN-REAL16: ; %bb.0: ; %entry
+; GCN-REAL16-NEXT: global_wb
+; GCN-REAL16-NEXT: v_nop
; GCN-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-REAL16-NEXT: v_mov_b16_e32 v1.l, 1
; GCN-REAL16-NEXT: flat_store_b16 v0, v1, s[0:1] scale_offset
@@ -325,6 +365,8 @@ entry:
define amdgpu_ps void @flat_store_b64_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_store_b64_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b64_e32 v[2:3], 1.0
; GCN-NEXT: flat_store_b64 v0, v[2:3], s[0:1] scale_offset
@@ -339,6 +381,8 @@ entry:
define amdgpu_ps void @flat_atomicrmw_b32_idxprom(ptr align 4 inreg %p, i32 %idx) {
; GCN-LABEL: flat_atomicrmw_b32_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: v_mov_b32_e32 v1, 1
; GCN-NEXT: flat_atomic_add_u32 v0, v1, s[0:1] scale_offset scope:SCOPE_SYS
@@ -353,6 +397,8 @@ entry:
define amdgpu_ps <2 x float> @flat_atomicrmw_b64_rtn_idxprom(ptr align 8 inreg %p, i32 %idx) {
; SDAG-LABEL: flat_atomicrmw_b64_rtn_idxprom:
; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -395,6 +441,8 @@ define amdgpu_ps <2 x float> @flat_atomicrmw_b64_rtn_idxprom(ptr align 8 inreg %
;
; GISEL-LABEL: flat_atomicrmw_b64_rtn_idxprom:
; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: v_mov_b32_e32 v2, v0
; GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
index 76477cd3af74a..2fddcece3e8b7 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-global.ll
@@ -12,6 +12,8 @@
define amdgpu_ps float @global_load_b32_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b32_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -32,6 +34,8 @@ entry:
define amdgpu_ps float @global_load_b32_idx32(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b32_idx32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -51,6 +55,8 @@ entry:
define amdgpu_ps float @global_load_b32_idxprom_wrong_stride(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b32_idxprom_wrong_stride:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -93,6 +99,8 @@ entry:
define amdgpu_ps float @global_load_b16_idxprom_ioffset(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b16_idxprom_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_u16 v0, v0, s[0:1] offset:32 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -116,6 +124,8 @@ entry:
define amdgpu_ps <2 x float> @global_load_b64_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b64_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -136,6 +146,8 @@ entry:
define amdgpu_ps <3 x float> @global_load_b96_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b96_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b96 v[0:2], v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -156,6 +168,8 @@ entry:
define amdgpu_ps <3 x float> @global_load_b96_idxpromi_ioffset(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b96_idxpromi_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b96 v[0:2], v0, s[0:1] offset:192 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -177,6 +191,8 @@ entry:
define amdgpu_ps <4 x float> @global_load_b128_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_load_b128_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b128 v[0:3], v0, s[0:1] scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -197,6 +213,8 @@ entry:
define amdgpu_ps float @global_load_b32_idxprom_range(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b32_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -222,6 +240,8 @@ entry:
define amdgpu_ps float @global_load_b32_idxprom_range_ioffset(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b32_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -250,6 +270,8 @@ entry:
define amdgpu_ps float @global_load_b8_idxprom_range_ioffset(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b8_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -278,6 +300,8 @@ entry:
define amdgpu_ps float @global_load_b16_idxprom_range(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b16_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -305,6 +329,8 @@ entry:
define amdgpu_ps float @global_load_b16_idxprom_range_ioffset(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b16_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -333,6 +359,8 @@ entry:
define amdgpu_ps <2 x float> @global_load_b64_idxprom_range(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b64_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -358,6 +386,8 @@ entry:
define amdgpu_ps <3 x float> @global_load_b96_idxprom_range(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b96_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -383,6 +413,8 @@ entry:
define amdgpu_ps <3 x float> @global_load_b96_idxprom_range_ioffset(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b96_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -409,6 +441,8 @@ entry:
define amdgpu_ps <4 x float> @global_load_b128_idxprom_range(ptr addrspace(1) align 4 inreg %p, ptr addrspace(1) align 4 %pp) {
; GFX1250-LABEL: global_load_b128_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -434,6 +468,8 @@ entry:
define amdgpu_ps void @global_store_b32_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_store_b32_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 1.0
; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1] scale_offset
@@ -454,6 +490,8 @@ entry:
define amdgpu_ps void @global_store_b16_idxprom(ptr addrspace(1) align 2 inreg %p, i32 %idx) {
; GFX1250-FAKE16-LABEL: global_store_b16_idxprom:
; GFX1250-FAKE16: ; %bb.0: ; %entry
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1] scale_offset
@@ -461,6 +499,8 @@ define amdgpu_ps void @global_store_b16_idxprom(ptr addrspace(1) align 2 inreg %
;
; GFX1250-TRUE16-LABEL: global_store_b16_idxprom:
; GFX1250-TRUE16: ; %bb.0: ; %entry
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, 1
; GFX1250-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1] scale_offset
@@ -487,6 +527,8 @@ entry:
define amdgpu_ps void @global_store_b64_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_store_b64_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 1.0
; GFX1250-NEXT: global_store_b64 v0, v[2:3], s[0:1] scale_offset
@@ -507,6 +549,8 @@ entry:
define amdgpu_ps void @global_atomicrmw_b32_idxprom(ptr addrspace(1) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_atomicrmw_b32_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scale_offset scope:SCOPE_SYS
@@ -527,6 +571,8 @@ entry:
define amdgpu_ps <2 x float> @global_atomicrmw_b64_rtn_idxprom(ptr addrspace(1) align 8 inreg %p, i32 %idx) {
; GFX1250-LABEL: global_atomicrmw_b64_rtn_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 1
; GFX1250-NEXT: global_atomic_add_u64 v[0:1], v0, v[2:3], s[0:1] scale_offset th:TH_ATOMIC_RETURN scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-scratch.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-scratch.ll
index cc9b4ffd67536..d9282a34fe11c 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-scratch.ll
@@ -11,6 +11,8 @@
define amdgpu_ps float @scratch_load_b32_alloca_idxprom(i32 %idx) {
; GFX1250-LABEL: scratch_load_b32_alloca_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -32,6 +34,8 @@ entry:
define amdgpu_ps float @scratch_load_b32_idxprom(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b32_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, s0 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -52,6 +56,8 @@ entry:
define amdgpu_ps float @scratch_load_b32_idx32(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b32_idx32:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, s0 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -71,6 +77,8 @@ entry:
define amdgpu_ps float @scratch_load_b32_idxprom_wrong_stride(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b32_idxprom_wrong_stride:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: scratch_load_b32 v0, v0, s0
@@ -93,6 +101,8 @@ entry:
define amdgpu_ps float @scratch_load_b16_idxprom_ioffset(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b16_idxprom_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_u16 v0, v0, s0 offset:32 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -116,6 +126,8 @@ entry:
define amdgpu_ps <2 x float> @scratch_load_b64_idxprom(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b64_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b64 v[0:1], v0, s0 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -136,6 +148,8 @@ entry:
define amdgpu_ps <3 x float> @scratch_load_b96_idxprom(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b96_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b96 v[0:2], v0, s0 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -156,6 +170,8 @@ entry:
define amdgpu_ps <3 x float> @scratch_load_b96_idxpromi_ioffset(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b96_idxpromi_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b96 v[0:2], v0, s0 offset:192 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -177,6 +193,8 @@ entry:
define amdgpu_ps <4 x float> @scratch_load_b128_idxprom(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_load_b128_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b128 v[0:3], v0, s0 scale_offset
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -197,6 +215,8 @@ entry:
define amdgpu_ps float @scratch_load_b32_idxprom_range(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b32_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -222,6 +242,8 @@ entry:
define amdgpu_ps float @scratch_load_b32_idxprom_range_ioffset(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b32_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -248,6 +270,8 @@ entry:
define amdgpu_ps float @scratch_load_b8_idxprom_range_ioffset(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b8_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -276,6 +300,8 @@ entry:
define amdgpu_ps float @scratch_load_b16_idxprom_range(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b16_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -303,6 +329,8 @@ entry:
define amdgpu_ps float @scratch_load_b16_idxprom_range_ioffset(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b16_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -331,6 +359,8 @@ entry:
define amdgpu_ps <2 x float> @scratch_load_b64_idxprom_range(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b64_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -358,6 +388,8 @@ entry:
define amdgpu_ps <3 x float> @scratch_load_b96_idxprom_range(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b96_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -383,6 +415,8 @@ entry:
define amdgpu_ps <3 x float> @scratch_load_b96_idxprom_range_ioffset(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b96_idxprom_range_ioffset:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -409,6 +443,8 @@ entry:
define amdgpu_ps <4 x float> @scratch_load_b128_idxprom_range(ptr addrspace(5) align 4 inreg %p, ptr addrspace(5) align 4 %pp) {
; GFX1250-LABEL: scratch_load_b128_idxprom_range:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: scratch_load_b32 v0, v0, off
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -434,6 +470,8 @@ entry:
define amdgpu_ps void @scratch_store_b32_idxprom(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_store_b32_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v1, 1.0
; GFX1250-NEXT: scratch_store_b32 v0, v1, s0 scale_offset
@@ -454,6 +492,8 @@ entry:
define amdgpu_ps void @scratch_store_b16_idxprom(ptr addrspace(5) align 2 inreg %p, i32 %idx) {
; GFX1250-FAKE16-LABEL: scratch_store_b16_idxprom:
; GFX1250-FAKE16: ; %bb.0: ; %entry
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 1
; GFX1250-FAKE16-NEXT: scratch_store_b16 v0, v1, s0 scale_offset
@@ -461,6 +501,8 @@ define amdgpu_ps void @scratch_store_b16_idxprom(ptr addrspace(5) align 2 inreg
;
; GFX1250-REAL16-LABEL: scratch_store_b16_idxprom:
; GFX1250-REAL16: ; %bb.0: ; %entry
+; GFX1250-REAL16-NEXT: global_wb
+; GFX1250-REAL16-NEXT: v_nop
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: v_mov_b16_e32 v1.l, 1
; GFX1250-REAL16-NEXT: scratch_store_b16 v0, v1, s0 scale_offset
@@ -487,6 +529,8 @@ entry:
define amdgpu_ps void @scratch_store_b64_idxprom(ptr addrspace(5) align 4 inreg %p, i32 %idx) {
; GFX1250-LABEL: scratch_store_b64_idxprom:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 1.0
; GFX1250-NEXT: scratch_store_b64 v0, v[2:3], s0 scale_offset
diff --git a/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll b/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
index 7375ca157bea9..7b13482ff8b8f 100644
--- a/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
+++ b/llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll
@@ -5,6 +5,8 @@
define amdgpu_ps float @s_load_b32_idxprom(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b32_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s0, s[0:1], s2 offset:0x0 scale_offset nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -22,6 +24,8 @@ entry:
define amdgpu_ps float @s_load_b32_idx32(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; SDAG-LABEL: s_load_b32_idx32:
; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: s_ashr_i32 s3, s2, 31
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -34,6 +38,8 @@ define amdgpu_ps float @s_load_b32_idx32(ptr addrspace(4) align 4 inreg %p, i32
;
; GISEL-LABEL: s_load_b32_idx32:
; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_ashr_i32 s3, s2, 31
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -53,6 +59,8 @@ entry:
define amdgpu_ps float @s_load_b32_idxprom_wrong_stride(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; SDAG-LABEL: s_load_b32_idxprom_wrong_stride:
; SDAG: ; %bb.0: ; %entry
+; SDAG-NEXT: global_wb
+; SDAG-NEXT: v_nop
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; SDAG-NEXT: s_mov_b32 s3, 0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -65,6 +73,8 @@ define amdgpu_ps float @s_load_b32_idxprom_wrong_stride(ptr addrspace(4) align 4
;
; GISEL-LABEL: s_load_b32_idxprom_wrong_stride:
; GISEL: ; %bb.0: ; %entry
+; GISEL-NEXT: global_wb
+; GISEL-NEXT: v_nop
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GISEL-NEXT: s_mov_b32 s3, 0
; GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -85,6 +95,8 @@ entry:
define amdgpu_ps float @s_load_b16_idxprom_ioffset(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b16_idxprom_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_u16 s0, s[0:1], s2 offset:0x20 scale_offset nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -103,6 +115,8 @@ entry:
define amdgpu_ps <2 x float> @s_load_b64_idxprom(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b64_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b64 s[4:5], s[0:1], s2 offset:0x0 scale_offset nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -118,6 +132,8 @@ entry:
define amdgpu_ps <3 x float> @s_load_b96_idxprom(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b96_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b96 s[4:6], s[0:1], s2 offset:0x0 scale_offset nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -134,6 +150,8 @@ entry:
define amdgpu_ps <4 x float> @s_load_b128_idxprom(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b128_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[4:7], s[0:1], s2 offset:0x0 scale_offset nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -150,6 +168,8 @@ entry:
define amdgpu_ps <8 x float> @s_load_b256_idxprom(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b256_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b256 s[4:11], s[0:1], s2 offset:0x0 scale_offset nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -168,6 +188,8 @@ entry:
define amdgpu_ps <16 x float> @s_load_b512_idxprom(ptr addrspace(4) align 4 inreg %p, i32 inreg %idx) {
; GCN-LABEL: s_load_b512_idxprom:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b512 s[4:19], s[0:1], s2 offset:0x0 scale_offset nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -190,6 +212,8 @@ entry:
define amdgpu_ps float @s_load_b32_idxprom_range(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b32_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -208,6 +232,8 @@ entry:
define amdgpu_ps float @s_load_b32_idxprom_range_ioffset(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b32_idxprom_range_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -229,6 +255,8 @@ entry:
define amdgpu_ps float @s_load_b8_idxprom_range_ioffset(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b8_idxprom_range_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -250,6 +278,8 @@ entry:
define amdgpu_ps float @s_load_b16_idxprom_range(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b16_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -270,6 +300,8 @@ entry:
define amdgpu_ps float @s_load_b16_idxprom_range_ioffset(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b16_idxprom_range_ioffset:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -291,6 +323,8 @@ entry:
define amdgpu_ps <2 x float> @s_load_b64_idxprom_range(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b64_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s4, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -309,6 +343,8 @@ entry:
define amdgpu_ps <3 x float> @s_load_b96_idxprom_range(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b96_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -328,6 +364,8 @@ entry:
define amdgpu_ps <4 x float> @s_load_b128_idxprom_range(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b128_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -347,6 +385,8 @@ entry:
define amdgpu_ps <8 x float> @s_load_b256_idxprom_range(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b256_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
@@ -368,6 +408,8 @@ entry:
define amdgpu_ps <16 x float> @s_load_b512_idxprom_range(ptr addrspace(4) align 4 inreg %p) {
; GCN-LABEL: s_load_b512_idxprom_range:
; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b32 s2, s[0:1], 0x0 nv
; GCN-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
index 9b57faa81d46f..f92f52d3020f0 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll
@@ -5,6 +5,8 @@
define amdgpu_kernel void @s_shl_v2i64(ptr addrspace(1) %out, <2 x i64> %lhs, <2 x i64> %rhs) #0 {
; GFX1250-LABEL: s_shl_v2i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -20,6 +22,8 @@ define amdgpu_kernel void @s_shl_v2i64(ptr addrspace(1) %out, <2 x i64> %lhs, <2
;
; GFX1251-LABEL: s_shl_v2i64:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b256 s[8:15], s[4:5], 0x34 nv
@@ -40,6 +44,8 @@ define amdgpu_kernel void @s_shl_v2i64(ptr addrspace(1) %out, <2 x i64> %lhs, <2
define amdgpu_kernel void @s_shl_v2i64_s_imm(ptr addrspace(1) %out, <2 x i64> %lhs) #0 {
; GFX1250-LABEL: s_shl_v2i64_s_imm:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
@@ -55,6 +61,8 @@ define amdgpu_kernel void @s_shl_v2i64_s_imm(ptr addrspace(1) %out, <2 x i64> %l
;
; GFX1251-LABEL: s_shl_v2i64_s_imm:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
@@ -75,6 +83,8 @@ define amdgpu_kernel void @s_shl_v2i64_s_imm(ptr addrspace(1) %out, <2 x i64> %l
define amdgpu_kernel void @s_shl_v2i64_imm_s(ptr addrspace(1) %out, <2 x i64> %rhs) #0 {
; GFX1250-LABEL: s_shl_v2i64_imm_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
@@ -90,6 +100,8 @@ define amdgpu_kernel void @s_shl_v2i64_imm_s(ptr addrspace(1) %out, <2 x i64> %r
;
; GFX1251-LABEL: s_shl_v2i64_imm_s:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_clause 0x1
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
@@ -110,6 +122,8 @@ define amdgpu_kernel void @s_shl_v2i64_imm_s(ptr addrspace(1) %out, <2 x i64> %r
define amdgpu_kernel void @v_shl_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GCN-LABEL: v_shl_v2i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v8, 0x3ff, v0
@@ -137,6 +151,8 @@ define amdgpu_kernel void @v_shl_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %
define amdgpu_kernel void @shl_v_s_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in, <2 x i64> %sgpr) #0 {
; GCN-LABEL: shl_v_s_v2i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -160,6 +176,8 @@ define amdgpu_kernel void @shl_v_s_v2i64(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @shl_s_v_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in, <2 x i64> %sgpr) #0 {
; GCN-LABEL: shl_s_v_v2i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -183,6 +201,8 @@ define amdgpu_kernel void @shl_s_v_v2i64(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @shl_imm_v_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GCN-LABEL: shl_imm_v_v2i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -206,6 +226,8 @@ define amdgpu_kernel void @shl_imm_v_v2i64(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @shl_v_imm_v2i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GCN-LABEL: shl_v_imm_v2i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v4, 0x3ff, v0
@@ -229,6 +251,8 @@ define amdgpu_kernel void @shl_v_imm_v2i64(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @v_shl_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GCN-LABEL: v_shl_v4i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -265,6 +289,8 @@ define amdgpu_kernel void @v_shl_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %
define amdgpu_kernel void @shl_v_imm_v4i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GCN-LABEL: shl_v_imm_v4i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -297,6 +323,8 @@ define amdgpu_kernel void @shl_v_imm_v4i64(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @v_shl_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GCN-LABEL: v_shl_v8i64:
; GCN: ; %bb.0:
+; GCN-NEXT: global_wb
+; GCN-NEXT: v_nop
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -345,6 +373,8 @@ define amdgpu_kernel void @v_shl_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %
define amdgpu_kernel void @v_shl_v16i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GFX1250-LABEL: v_shl_v16i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -405,6 +435,8 @@ define amdgpu_kernel void @v_shl_v16i64(ptr addrspace(1) %out, ptr addrspace(1)
;
; GFX1251-LABEL: v_shl_v16i64:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -479,6 +511,8 @@ define amdgpu_kernel void @v_shl_v16i64(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @v_shl_v32i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
; GFX1250-LABEL: v_shl_v32i64:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
@@ -589,6 +623,8 @@ define amdgpu_kernel void @v_shl_v32i64(ptr addrspace(1) %out, ptr addrspace(1)
;
; GFX1251-LABEL: v_shl_v32i64:
; GFX1251: ; %bb.0:
+; GFX1251-NEXT: global_wb
+; GFX1251-NEXT: v_nop
; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0
diff --git a/llvm/test/CodeGen/AMDGPU/spillv16Kernel.ll b/llvm/test/CodeGen/AMDGPU/spillv16Kernel.ll
index e7f6c4406676e..2ef4d1f37a477 100644
--- a/llvm/test/CodeGen/AMDGPU/spillv16Kernel.ll
+++ b/llvm/test/CodeGen/AMDGPU/spillv16Kernel.ll
@@ -5,6 +5,8 @@
define amdgpu_kernel void @spill_i16_alu() #0 {
; GFX1250-TRUE16-LABEL: spill_i16_alu:
; GFX1250-TRUE16: ; %bb.0: ; %entry
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: scratch_load_u16 v0, off, off scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -23,6 +25,8 @@ define amdgpu_kernel void @spill_i16_alu() #0 {
;
; GFX1250-FAKE16-LABEL: spill_i16_alu:
; GFX1250-FAKE16: ; %bb.0: ; %entry
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: scratch_load_u16 v0, off, off scope:SCOPE_SYS
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/sub_u64.ll b/llvm/test/CodeGen/AMDGPU/sub_u64.ll
index ffd0b5c793580..9138c69d434ca 100644
--- a/llvm/test/CodeGen/AMDGPU/sub_u64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub_u64.ll
@@ -13,6 +13,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_vv(i64 %a, i64 %b) {
;
; GFX1250-LABEL: test_sub_u64_vv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], v[0:1], v[2:3]
; GFX1250-NEXT: ; return to shader part epilog
@@ -31,6 +33,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_vs(i64 %a, i64 inreg %b) {
;
; GFX1250-LABEL: test_sub_u64_vs:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_sub_nc_u64_e64 v[0:1], v[0:1], s[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -49,6 +53,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_sv(i64 inreg %a, i64 %b) {
;
; GFX1250-LABEL: test_sub_u64_sv:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], s[0:1], v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -67,6 +73,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_ss(i64 inreg %a, i64 inreg %b) {
;
; GFX1250-LABEL: test_sub_u64_ss:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sub_nc_u64 s[0:1], s[0:1], s[2:3]
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -87,6 +95,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_inline_lit_v(i64 %a) {
;
; GFX1250-LABEL: test_sub_u64_inline_lit_v:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], 5, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -105,6 +115,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_v_inline_lit(i64 %a) {
;
; GFX1250-LABEL: test_sub_u64_v_inline_lit:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], -5, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -123,6 +135,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_small_imm_v(i64 %a) {
;
; GFX1250-LABEL: test_sub_u64_small_imm_v:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], 0x1f4, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -141,6 +155,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_64bit_imm_v(i64 %a) {
;
; GFX1250-LABEL: test_sub_u64_64bit_imm_v:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], 0x13b9ac9ff, v[0:1]
; GFX1250-NEXT: ; return to shader part epilog
@@ -159,6 +175,8 @@ define amdgpu_ps <2 x float> @test_sub_u64_small_imm_s(i64 inreg %a) {
;
; GFX1250-LABEL: test_sub_u64_small_imm_s:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_sub_nc_u64 s[0:1], 0x1f4, s[0:1]
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll b/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
index 4d6ff1083492a..dc6004153ae4a 100644
--- a/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
@@ -20,6 +20,8 @@ define amdgpu_kernel void @v_ashr_pk_i8_i32(ptr addrspace(1) %out, i32 %src0, i3
;
; GFX1250-TRUE16-LABEL: v_ashr_pk_i8_i32:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
; GFX1250-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
@@ -35,6 +37,8 @@ define amdgpu_kernel void @v_ashr_pk_i8_i32(ptr addrspace(1) %out, i32 %src0, i3
;
; GFX1250-FAKE16-LABEL: v_ashr_pk_i8_i32:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
; GFX1250-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
@@ -105,6 +109,8 @@ define amdgpu_kernel void @v_ashr_pk_u8_i32(ptr addrspace(1) %out, i32 %src0, i3
;
; GFX1250-TRUE16-LABEL: v_ashr_pk_u8_i32:
; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: global_wb
+; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
; GFX1250-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
@@ -120,6 +126,8 @@ define amdgpu_kernel void @v_ashr_pk_u8_i32(ptr addrspace(1) %out, i32 %src0, i3
;
; GFX1250-FAKE16-LABEL: v_ashr_pk_u8_i32:
; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: global_wb
+; GFX1250-FAKE16-NEXT: v_nop
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
; GFX1250-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
diff --git a/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll b/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll
index 5e000938fae68..0d9f6dbefd591 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll
@@ -12,6 +12,8 @@ define amdgpu_ps void @intrinsic_store_system_scope(i32 %val, <4 x i32> inreg %r
;
; GFX1250-SDAG-LABEL: intrinsic_store_system_scope:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: buffer_store_b32 v0, v[2:3], s[0:3], s4 idxen offen scope:SCOPE_SYS
@@ -19,6 +21,8 @@ define amdgpu_ps void @intrinsic_store_system_scope(i32 %val, <4 x i32> inreg %r
;
; GFX1250-GISEL-LABEL: intrinsic_store_system_scope:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: buffer_store_b32 v0, v[4:5], s[0:3], s4 idxen offen scope:SCOPE_SYS
@@ -36,6 +40,8 @@ define amdgpu_ps void @generic_store_volatile(i32 %val, ptr addrspace(1) %out) {
;
; GFX1250-SDAG-LABEL: generic_store_volatile:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: global_store_b32 v[2:3], v0, off scope:SCOPE_SYS
@@ -44,6 +50,8 @@ define amdgpu_ps void @generic_store_volatile(i32 %val, ptr addrspace(1) %out) {
;
; GFX1250-GISEL-LABEL: generic_store_volatile:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: global_store_b32 v[4:5], v0, off scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
index 04fad1cb91649..bafab2da321b7 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll
@@ -5,6 +5,8 @@
define amdgpu_kernel void @single_atomic_rmw(ptr addrspace(1) %ptr) {
; GFX1250-LABEL: single_atomic_rmw:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -32,6 +34,8 @@ define amdgpu_kernel void @single_atomic_rmw(ptr addrspace(1) %ptr) {
define amdgpu_kernel void @atomic_rmw_back_to_back(ptr addrspace(1) %ptr) {
; GFX1250-LABEL: atomic_rmw_back_to_back:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -83,6 +87,8 @@ define amdgpu_kernel void @atomic_rmw_back_to_back(ptr addrspace(1) %ptr) {
define amdgpu_kernel void @atomic_rmw_with_alu(ptr addrspace(1) %ptr, i32 %a, i32 %b) {
; GFX1250-LABEL: atomic_rmw_with_alu:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -144,6 +150,8 @@ define amdgpu_kernel void @atomic_rmw_with_alu(ptr addrspace(1) %ptr, i32 %a, i3
define amdgpu_kernel void @atomic_rmw_broken_by_global_load(ptr addrspace(1) %ptr, ptr addrspace(1) %load_ptr) {
; GFX1250-LABEL: atomic_rmw_broken_by_global_load:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -200,6 +208,8 @@ define amdgpu_kernel void @atomic_rmw_broken_by_global_load(ptr addrspace(1) %pt
define amdgpu_kernel void @atomic_rmw_broken_by_global_store(ptr addrspace(1) %ptr, ptr addrspace(1) %store_ptr) {
; GFX1250-LABEL: atomic_rmw_broken_by_global_store:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -259,6 +269,8 @@ define amdgpu_kernel void @atomic_rmw_broken_by_global_store(ptr addrspace(1) %p
define amdgpu_kernel void @atomic_rmw_broken_by_flat_load(ptr addrspace(1) %ptr, ptr %flat_ptr) {
; GFX1250-LABEL: atomic_rmw_broken_by_flat_load:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -314,6 +326,8 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_load(ptr addrspace(1) %ptr,
define amdgpu_kernel void @atomic_rmw_broken_by_flat_store(ptr addrspace(1) %ptr, ptr %flat_ptr) {
; GFX1250-LABEL: atomic_rmw_broken_by_flat_store:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -372,6 +386,8 @@ define amdgpu_kernel void @atomic_rmw_broken_by_flat_store(ptr addrspace(1) %ptr
define amdgpu_kernel void @atomic_rmw_broken_by_smem_load(ptr addrspace(1) %ptr, ptr addrspace(4) %const_ptr) {
; GFX1250-LABEL: atomic_rmw_broken_by_smem_load:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -430,6 +446,8 @@ define amdgpu_kernel void @atomic_rmw_broken_by_smem_load(ptr addrspace(1) %ptr,
define amdgpu_kernel void @atomic_rmw_broken_by_atomic_store(ptr addrspace(1) %ptr, ptr addrspace(1) %store_ptr) {
; GFX1250-LABEL: atomic_rmw_broken_by_atomic_store:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -495,6 +513,8 @@ define amdgpu_kernel void @atomic_rmw_broken_by_atomic_store(ptr addrspace(1) %p
define amdgpu_kernel void @atomic_rmw_with_lds_load(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
; GFX1250-LABEL: atomic_rmw_with_lds_load:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -551,6 +571,8 @@ define amdgpu_kernel void @atomic_rmw_with_lds_load(ptr addrspace(1) %ptr, ptr a
define amdgpu_kernel void @atomic_rmw_with_lds_store(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
; GFX1250-LABEL: atomic_rmw_with_lds_store:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -609,6 +631,8 @@ define amdgpu_kernel void @atomic_rmw_with_lds_store(ptr addrspace(1) %ptr, ptr
define amdgpu_kernel void @atomic_rmw_with_flat_lds_load(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
; GFX1250-LABEL: atomic_rmw_with_flat_lds_load:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -679,6 +703,8 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_load(ptr addrspace(1) %ptr,
define amdgpu_kernel void @atomic_rmw_with_flat_lds_store(ptr addrspace(1) %ptr, ptr addrspace(3) %lds_ptr) {
; GFX1250-LABEL: atomic_rmw_with_flat_lds_store:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -752,6 +778,8 @@ define amdgpu_kernel void @atomic_rmw_with_flat_lds_store(ptr addrspace(1) %ptr,
define amdgpu_kernel void @atomic_rmw_borken_by_async_lds_copy(ptr addrspace(1) %ptr, ptr addrspace(1) %src, ptr addrspace(3) %dst) {
; GFX1250-LABEL: atomic_rmw_borken_by_async_lds_copy:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b64 s[2:3], s[4:5]
; GFX1250-NEXT: s_load_b64 s[0:1], s[2:3], 0x0 nv
@@ -819,6 +847,8 @@ define amdgpu_kernel void @multiple_atomic_rmw_blocks(ptr addrspace(1) %ptr1, pt
; First block
; GFX1250-LABEL: multiple_atomic_rmw_blocks:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -887,6 +917,8 @@ define amdgpu_kernel void @multiple_atomic_rmw_blocks(ptr addrspace(1) %ptr1, pt
define amdgpu_kernel void @different_atomic_ops(ptr addrspace(1) %ptr) {
; GFX1250-LABEL: different_atomic_ops:
; GFX1250: ; %bb.0:
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -950,6 +982,8 @@ define amdgpu_kernel void @different_atomic_ops(ptr addrspace(1) %ptr) {
define amdgpu_kernel void @atomic_rmw_across_basic_blocks(ptr addrspace(1) %ptr, i32 %cond) {
; GFX1250-LABEL: atomic_rmw_across_basic_blocks:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1039,6 +1073,8 @@ exit:
define amdgpu_kernel void @atomic_rmw_in_loop(ptr addrspace(1) %ptr, i32 %n) {
; GFX1250-LABEL: atomic_rmw_in_loop:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1125,6 +1161,8 @@ exit:
define amdgpu_kernel void @atomic_rmw_with_branch(ptr addrspace(1) %ptr, i32 %cond) {
; GFX1250-LABEL: atomic_rmw_with_branch:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1276,6 +1314,8 @@ merge:
define amdgpu_kernel void @atomic_rmw_fallthrough(ptr addrspace(1) %ptr) {
; GFX1250-LABEL: atomic_rmw_fallthrough:
; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir b/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir
index e3c9226acded8..8c405283e25f4 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir
@@ -12,6 +12,8 @@ body: |
; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; GCN-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -75,6 +77,8 @@ body: |
; GCN-NEXT: successors: %bb.1(0x80000000)
; GCN-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETPC_B64 $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
@@ -106,6 +110,8 @@ body: |
; GCN-LABEL: name: drain_trap
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_TRAP 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -127,6 +133,8 @@ body: |
; GCN-LABEL: name: drain_getreg
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0 = S_GETREG_B32 1, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -148,6 +156,8 @@ body: |
; GCN-LABEL: name: drain_setreg
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_B32 $sgpr0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -169,6 +179,8 @@ body: |
; GCN-LABEL: name: drain_setreg_mode
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_B32_mode $sgpr0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -190,6 +202,8 @@ body: |
; GCN-LABEL: name: drain_setreg_imm32
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_IMM32_B32 0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -211,6 +225,8 @@ body: |
; GCN-LABEL: name: drain_setreg_imm32_mode
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_IMM32_B32_mode 0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -232,6 +248,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SENDMSG 1, implicit $exec, implicit $m0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -253,6 +271,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg_halt
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SENDMSGHALT 1, implicit $exec, implicit $m0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -274,6 +294,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg_rtn_b32
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0 = S_SENDMSG_RTN_B32 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -295,6 +317,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg_rtn_b64
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0_sgpr1 = S_SENDMSG_RTN_B64 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -316,6 +340,8 @@ body: |
; GCN-LABEL: name: drain_barrier_wait
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_WAIT 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -337,6 +363,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal_m0
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_M0 implicit $m0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -358,6 +386,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_IMM 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -379,6 +409,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal_isfirst_m0
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_ISFIRST_M0 implicit $m0, implicit-def $scc, implicit $scc
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -400,6 +432,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal_isfirst
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_ISFIRST_IMM 1, implicit-def $scc, implicit $scc
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir b/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
index 1261b22f1cbfa..bf7c190a11dbe 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
@@ -12,6 +12,8 @@ body: |
; GCN-LABEL: name: vmem_scratch_load
; GCN: liveins: $vgpr0
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr1 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
@@ -34,6 +36,8 @@ body: |
; GCN-LABEL: name: vmem_buffer_load_dword_offset
; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
@@ -56,6 +60,8 @@ body: |
; GCN-LABEL: name: vmem_buffer_load_addr
; GCN: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_ADDR64 $vgpr0_vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
@@ -78,6 +84,8 @@ body: |
; GCN-LABEL: name: vmem_flat_load
; GCN: liveins: $vgpr0, $vgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
@@ -100,6 +108,8 @@ body: |
; GCN-LABEL: name: vmem_global_load
; GCN: liveins: $vgpr0, $vgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
@@ -122,6 +132,8 @@ body: |
; GCN-LABEL: name: vmem_global_store
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_XCNT 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -181,6 +193,8 @@ body: |
; GCN-LABEL: name: smem_load
; GCN: liveins: $sgpr2_sgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr2_sgpr3, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: S_WAIT_XCNT 0
; GCN-NEXT: $sgpr2 = S_MOV_B32 0
@@ -199,6 +213,8 @@ body: |
; GCN-LABEL: name: smem_store
; GCN: liveins: $sgpr0, $sgpr2, $sgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr0, $sgpr2_sgpr3, 0, 0
; GCN-NEXT: S_WAIT_XCNT 0
; GCN-NEXT: $sgpr3 = S_MOV_B32 0
@@ -219,6 +235,8 @@ body: |
; GCN-LABEL: name: vmem_load_group
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -250,6 +268,8 @@ body: |
; GCN-LABEL: name: vmem_store_group
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -286,6 +306,8 @@ body: |
; GCN-LABEL: name: smem_load_group
; GCN: liveins: $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: $sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
@@ -310,6 +332,8 @@ body: |
; GCN-LABEL: name: smem_store_group
; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr2, $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr3, $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr4, $sgpr0_sgpr1, 0, 0
@@ -338,6 +362,8 @@ body: |
; GCN-LABEL: name: vmem_loads_with_an_intervening_nop
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -369,6 +395,8 @@ body: |
; GCN-LABEL: name: vmem_contiguous_loads_with_an_intervening_store
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -401,6 +429,8 @@ body: |
; GCN-LABEL: name: vmem_stores_with_intervening_nop
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -441,6 +471,8 @@ body: |
; GCN-LABEL: name: vmem_contiguous_stores_with_an_intervening_load
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -483,6 +515,8 @@ body: |
; GCN-LABEL: name: atomic_op
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr2 = BUFFER_ATOMIC_ADD_ADDR64_RTN $vgpr2, $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 1, implicit $exec :: (load store (s32), addrspace 1)
; GCN-NEXT: GLOBAL_ATOMIC_ADD_F32 $vgpr4_vgpr5, killed renamable $vgpr3, 0, 0, implicit $exec :: (load store syncscope("agent-one-as") monotonic monotonic (s32), addrspace 1)
; GCN-NEXT: $vgpr6 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
@@ -511,6 +545,8 @@ body: |
; GCN-LABEL: name: smem_xcnt_insertion_with_pending_vmem_event
; GCN: liveins: $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 2, implicit $exec
@@ -547,6 +583,8 @@ body: |
; GCN-LABEL: name: vmem_group_reg_dependency_with_prior_instruction
; GCN: liveins: $vgpr4, $vgpr5
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -575,6 +613,8 @@ body: |
; GCN-LABEL: name: multiple_xcnt_insertion_in_group
; GCN: liveins: $vgpr3, $vgpr4, $vgpr5
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -607,6 +647,8 @@ body: |
; GCN-LABEL: name: xcnt_event_post_load_group
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -649,6 +691,8 @@ body: |
; GCN-LABEL: name: xcnt_event_post_store_group
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -703,6 +747,8 @@ body: |
; GCN-LABEL: name: load_store_switching
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr5, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -744,6 +790,8 @@ body: |
; GCN-LABEL: name: dual_mov
; GCN: liveins: $sgpr0, $sgpr1, $vgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr2 = SCRATCH_LOAD_DWORD $vgpr1, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr1, $vgpr2 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
@@ -763,7 +811,9 @@ machineFunctionInfo:
body: |
bb.0:
; GCN-LABEL: name: ds_load_store
- ; GCN: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+ ; GCN: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: $vgpr0 = DS_READ_B32_gfx9 killed $vgpr1, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) undef`, addrspace 3)
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 2, implicit $exec
; GCN-NEXT: S_WAIT_DSCNT 0
@@ -787,6 +837,8 @@ body: |
; GCN-LABEL: name: xcnt_max
; GCN: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
@@ -933,6 +985,8 @@ body: |
; GCN-LABEL: name: wait_kmcnt_with_outstanding_vmem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_KMCNT 0
@@ -956,6 +1010,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -996,6 +1052,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1036,6 +1094,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1080,6 +1140,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1125,6 +1187,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1172,6 +1236,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1216,6 +1282,8 @@ body: |
; GCN-LABEL: name: wait_loadcnt_with_outstanding_smem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -1239,6 +1307,8 @@ body: |
; GCN-LABEL: name: overwrite_vgpr_after_smem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
@@ -1258,6 +1328,8 @@ body: |
; GCN-LABEL: name: overwrite_sgpr_after_vmem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0 = S_MOV_B32 0
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
index d96228141956f..2897ec88d94b5 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
@@ -7,6 +7,8 @@
define amdgpu_kernel void @ds_prefetch_flushed(ptr addrspace(3) %lds, ptr addrspace(1) %out, i32 %n) {
; CHECK-LABEL: ds_prefetch_flushed:
; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir
index 39068b42fcf3a..5247bfe68226c 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir
@@ -16,6 +16,8 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $sgpr0, $vgpr0
; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; CHECK-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr28 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr32 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
index 7f98f8cf8de06..709f4e1a9d4fc 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
@@ -7,6 +7,8 @@
define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrspace(1) %out, i32 %n) {
; CHECK-LABEL: ds_prefetch_pattern:
; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: global_wb
+; CHECK-NEXT: v_nop
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
index 5b8b5207b1eee..0f2314264c30a 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s -check-prefix=GFX1250
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -run-pass=si-insert-waitcnts -o - %s | FileCheck %s -check-prefix=GFX1200
# Test for the fix that removed the incorrect S_BARRIER check for DS stores.
# Previously, the code would reset SeenDSStoreInCurrMBB when encountering an
@@ -23,31 +23,55 @@ tracksRegLiveness: true
machineFunctionInfo:
isEntryFunction: true
body: |
- ; CHECK-LABEL: name: ds_store_barrier_no_preheader_flush
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; GFX1250-LABEL: name: ds_store_barrier_no_preheader_flush
+ ; GFX1250: bb.0:
+ ; GFX1250-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GFX1250-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; GFX1250-NEXT: S_BRANCH %bb.1
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.1:
+ ; GFX1250-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1250-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr10
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: S_WAIT_DSCNT 0
+ ; GFX1250-NEXT: $vgpr30 = V_ADD_F32_e32 $vgpr10, $vgpr1, implicit $mode, implicit $exec
+ ; GFX1250-NEXT: DS_WRITE_B32 $vgpr2, $vgpr30, 0, 0, implicit $m0, implicit $exec
+ ; GFX1250-NEXT: S_BARRIER
+ ; GFX1250-NEXT: $sgpr0 = S_ADD_I32 $sgpr0, -1, implicit-def $scc
+ ; GFX1250-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; GFX1250-NEXT: S_BRANCH %bb.2
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: bb.2:
+ ; GFX1250-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1200-LABEL: name: ds_store_barrier_no_preheader_flush
+ ; GFX1200: bb.0:
+ ; GFX1200-NEXT: successors: %bb.1(0x80000000)
+ ; GFX1200-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+ ; GFX1200-NEXT: {{ $}}
+ ; GFX1200-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
+ ; GFX1200-NEXT: S_BRANCH %bb.1
+ ; GFX1200-NEXT: {{ $}}
+ ; GFX1200-NEXT: bb.1:
+ ; GFX1200-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; GFX1200-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr10
+ ; GFX1200-NEXT: {{ $}}
+ ; GFX1200-NEXT: S_WAIT_DSCNT 0
+ ; GFX1200-NEXT: $vgpr30 = V_ADD_F32_e32 $vgpr10, $vgpr1, implicit $mode, implicit $exec
+ ; GFX1200-NEXT: DS_WRITE_B32 $vgpr2, $vgpr30, 0, 0, implicit $m0, implicit $exec
+ ; GFX1200-NEXT: S_BARRIER
+ ; GFX1200-NEXT: $sgpr0 = S_ADD_I32 $sgpr0, -1, implicit-def $scc
+ ; GFX1200-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
+ ; GFX1200-NEXT: S_BRANCH %bb.2
+ ; GFX1200-NEXT: {{ $}}
+ ; GFX1200-NEXT: bb.2:
+ ; GFX1200-NEXT: S_ENDPGM 0
; Verify NO S_WAIT_DSCNT in preheader - the wait must be inside the loop
- ; CHECK-NOT: S_WAIT_DSCNT
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr10
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: S_WAIT_DSCNT 0
- ; CHECK-NEXT: $vgpr30 = V_ADD_F32_e32 $vgpr10, $vgpr1, implicit $mode, implicit $exec
- ; CHECK-NEXT: DS_WRITE_B32 $vgpr2, $vgpr30, 0, 0, implicit $m0, implicit $exec
; With BackOffBarrier, no S_WAIT_DSCNT needed before S_BARRIER
- ; CHECK-NEXT: S_BARRIER
- ; CHECK-NEXT: $sgpr0 = S_ADD_I32 $sgpr0, -1, implicit-def $scc
- ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit $scc
- ; CHECK-NEXT: S_BRANCH %bb.2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: S_ENDPGM 0
bb.0:
successors: %bb.1
liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir
index dbab9c95c498b..23510a309434e 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir
@@ -15,6 +15,8 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $sgpr0, $vgpr0
; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; CHECK-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr14_vgpr15_vgpr16_vgpr17 = DS_READ_B128 $vgpr0, 16, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr18_vgpr19_vgpr20_vgpr21 = DS_READ_B128 $vgpr0, 32, 0, implicit $m0, implicit $exec
@@ -81,6 +83,8 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $sgpr0, $vgpr0
; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec, implicit $exec
; CHECK-NEXT: $vgpr70_vgpr71_vgpr72_vgpr73 = DS_READ_B128 $vgpr0, 240, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr66_vgpr67_vgpr68_vgpr69 = DS_READ_B128 $vgpr0, 224, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr62_vgpr63_vgpr64_vgpr65 = DS_READ_B128 $vgpr0, 208, 0, implicit $m0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll b/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
index 909d13fcc3c2c..31b91da445277 100644
--- a/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
+++ b/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
@@ -7855,6 +7855,8 @@ define amdgpu_cs void @call_from_entry(<8 x float> %x, ptr %p) #0 {
;
; GFX1250-DAGISEL-LABEL: call_from_entry:
; GFX1250-DAGISEL: ; %bb.0:
+; GFX1250-DAGISEL-NEXT: global_wb
+; GFX1250-DAGISEL-NEXT: v_nop
; GFX1250-DAGISEL-NEXT: s_mov_b32 s32, 0
; GFX1250-DAGISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-DAGISEL-NEXT: s_mov_b64 s[0:1], callee at abs64
diff --git a/llvm/test/CodeGen/AMDGPU/workgroup-id-in-arch-sgprs.ll b/llvm/test/CodeGen/AMDGPU/workgroup-id-in-arch-sgprs.ll
index e65689c5a4124..ad48e8414acdc 100644
--- a/llvm/test/CodeGen/AMDGPU/workgroup-id-in-arch-sgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/workgroup-id-in-arch-sgprs.ll
@@ -27,6 +27,8 @@ define amdgpu_kernel void @workgroup_id_x(ptr addrspace(1) %ptrx) {
;
; GFX1250-SDAG-LABEL: workgroup_id_x:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT: s_bfe_u32 s2, ttmp6, 0x4000c
@@ -46,6 +48,8 @@ define amdgpu_kernel void @workgroup_id_x(ptr addrspace(1) %ptrx) {
;
; GFX1250-GISEL-LABEL: workgroup_id_x:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_bfe_u32 s2, ttmp6, 0x4000c
@@ -103,6 +107,8 @@ define amdgpu_kernel void @workgroup_id_xy(ptr addrspace(1) %ptrx, ptr addrspace
;
; GFX1250-SDAG-LABEL: workgroup_id_xy:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-SDAG-NEXT: s_bfe_u32 s6, ttmp6, 0x40010
@@ -131,6 +137,8 @@ define amdgpu_kernel void @workgroup_id_xy(ptr addrspace(1) %ptrx, ptr addrspace
;
; GFX1250-GISEL-LABEL: workgroup_id_xy:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_bfe_u32 s6, ttmp6, 0x4000c
; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
@@ -214,6 +222,8 @@ define amdgpu_kernel void @workgroup_id_xyz(ptr addrspace(1) %ptrx, ptr addrspac
;
; GFX1250-SDAG-LABEL: workgroup_id_xyz:
; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_bfe_u32 s0, ttmp6, 0x40014
; GFX1250-SDAG-NEXT: s_lshr_b32 s8, ttmp7, 16
@@ -253,6 +263,8 @@ define amdgpu_kernel void @workgroup_id_xyz(ptr addrspace(1) %ptrx, ptr addrspac
;
; GFX1250-GISEL-LABEL: workgroup_id_xyz:
; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_bfe_u32 s0, ttmp6, 0x4000c
; GFX1250-GISEL-NEXT: s_and_b32 s1, ttmp6, 15
>From 96a516e6c25bc8d0fe4312eb09021e8742184d32 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad at amd.com>
Date: Thu, 9 Jul 2026 17:27:56 +0100
Subject: [PATCH 2/2] Remove extra implicit operands
---
llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 6 +-
llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir | 68 ++++-----
llvm/test/CodeGen/AMDGPU/wait-xcnt.mir | 144 +++++++++---------
.../waitcnt-loop-ds-prefetch-flushed.mir | 4 +-
.../AMDGPU/waitcnt-loop-ds-store-barrier.mir | 4 +-
llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir | 8 +-
6 files changed, 116 insertions(+), 118 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index eb076b9a84c20..5996cdf42a956 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -3635,10 +3635,8 @@ bool SIInsertWaitcnts::run() {
// V_NOP
MachineBasicBlock::iterator I = EntryBB.begin();
BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::GLOBAL_WB))
- .addImm(AMDGPU::CPol::SCOPE_CU)
- .addReg(AMDGPU::EXEC, RegState::Implicit);
- BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::V_NOP_e32))
- .addReg(AMDGPU::EXEC, RegState::Implicit);
+ .addImm(AMDGPU::CPol::SCOPE_CU);
+ BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::V_NOP_e32));
Modified = true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir b/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir
index 8c405283e25f4..c95c09dc17f65 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir
@@ -12,8 +12,8 @@ body: |
; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; GCN-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -77,8 +77,8 @@ body: |
; GCN-NEXT: successors: %bb.1(0x80000000)
; GCN-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETPC_B64 $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
@@ -110,8 +110,8 @@ body: |
; GCN-LABEL: name: drain_trap
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_TRAP 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -133,8 +133,8 @@ body: |
; GCN-LABEL: name: drain_getreg
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0 = S_GETREG_B32 1, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -156,8 +156,8 @@ body: |
; GCN-LABEL: name: drain_setreg
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_B32 $sgpr0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -179,8 +179,8 @@ body: |
; GCN-LABEL: name: drain_setreg_mode
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_B32_mode $sgpr0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -202,8 +202,8 @@ body: |
; GCN-LABEL: name: drain_setreg_imm32
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_IMM32_B32 0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -225,8 +225,8 @@ body: |
; GCN-LABEL: name: drain_setreg_imm32_mode
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SETREG_IMM32_B32_mode 0, 1, implicit-def $mode, implicit $mode
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -248,8 +248,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SENDMSG 1, implicit $exec, implicit $m0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -271,8 +271,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg_halt
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_SENDMSGHALT 1, implicit $exec, implicit $m0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -294,8 +294,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg_rtn_b32
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0 = S_SENDMSG_RTN_B32 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -317,8 +317,8 @@ body: |
; GCN-LABEL: name: drain_sendmsg_rtn_b64
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0_sgpr1 = S_SENDMSG_RTN_B64 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -340,8 +340,8 @@ body: |
; GCN-LABEL: name: drain_barrier_wait
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_WAIT 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -363,8 +363,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal_m0
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_M0 implicit $m0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -386,8 +386,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_IMM 1
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -409,8 +409,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal_isfirst_m0
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $m0, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_ISFIRST_M0 implicit $m0, implicit-def $scc, implicit $scc
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -432,8 +432,8 @@ body: |
; GCN-LABEL: name: drain_barrier_signal_isfirst
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_BARRIER_SIGNAL_ISFIRST_IMM 1, implicit-def $scc, implicit $scc
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir b/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
index bf7c190a11dbe..4e0f7ccce2727 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
@@ -12,8 +12,8 @@ body: |
; GCN-LABEL: name: vmem_scratch_load
; GCN: liveins: $vgpr0
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr1 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
@@ -36,8 +36,8 @@ body: |
; GCN-LABEL: name: vmem_buffer_load_dword_offset
; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
@@ -60,8 +60,8 @@ body: |
; GCN-LABEL: name: vmem_buffer_load_addr
; GCN: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_ADDR64 $vgpr0_vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
@@ -84,8 +84,8 @@ body: |
; GCN-LABEL: name: vmem_flat_load
; GCN: liveins: $vgpr0, $vgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
@@ -108,8 +108,8 @@ body: |
; GCN-LABEL: name: vmem_global_load
; GCN: liveins: $vgpr0, $vgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
@@ -132,8 +132,8 @@ body: |
; GCN-LABEL: name: vmem_global_store
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_XCNT 0
; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
@@ -193,8 +193,8 @@ body: |
; GCN-LABEL: name: smem_load
; GCN: liveins: $sgpr2_sgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr2_sgpr3, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: S_WAIT_XCNT 0
; GCN-NEXT: $sgpr2 = S_MOV_B32 0
@@ -213,8 +213,8 @@ body: |
; GCN-LABEL: name: smem_store
; GCN: liveins: $sgpr0, $sgpr2, $sgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr0, $sgpr2_sgpr3, 0, 0
; GCN-NEXT: S_WAIT_XCNT 0
; GCN-NEXT: $sgpr3 = S_MOV_B32 0
@@ -235,8 +235,8 @@ body: |
; GCN-LABEL: name: vmem_load_group
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -268,8 +268,8 @@ body: |
; GCN-LABEL: name: vmem_store_group
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -306,8 +306,8 @@ body: |
; GCN-LABEL: name: smem_load_group
; GCN: liveins: $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: $sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
@@ -332,8 +332,8 @@ body: |
; GCN-LABEL: name: smem_store_group
; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr2, $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr3, $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_STORE_DWORD_IMM $sgpr4, $sgpr0_sgpr1, 0, 0
@@ -362,8 +362,8 @@ body: |
; GCN-LABEL: name: vmem_loads_with_an_intervening_nop
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -395,8 +395,8 @@ body: |
; GCN-LABEL: name: vmem_contiguous_loads_with_an_intervening_store
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -429,8 +429,8 @@ body: |
; GCN-LABEL: name: vmem_stores_with_intervening_nop
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -471,8 +471,8 @@ body: |
; GCN-LABEL: name: vmem_contiguous_stores_with_an_intervening_load
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -515,8 +515,8 @@ body: |
; GCN-LABEL: name: atomic_op
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr2 = BUFFER_ATOMIC_ADD_ADDR64_RTN $vgpr2, $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 1, implicit $exec :: (load store (s32), addrspace 1)
; GCN-NEXT: GLOBAL_ATOMIC_ADD_F32 $vgpr4_vgpr5, killed renamable $vgpr3, 0, 0, implicit $exec :: (load store syncscope("agent-one-as") monotonic monotonic (s32), addrspace 1)
; GCN-NEXT: $vgpr6 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
@@ -545,8 +545,8 @@ body: |
; GCN-LABEL: name: smem_xcnt_insertion_with_pending_vmem_event
; GCN: liveins: $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 2, implicit $exec
@@ -583,8 +583,8 @@ body: |
; GCN-LABEL: name: vmem_group_reg_dependency_with_prior_instruction
; GCN: liveins: $vgpr4, $vgpr5
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -613,8 +613,8 @@ body: |
; GCN-LABEL: name: multiple_xcnt_insertion_in_group
; GCN: liveins: $vgpr3, $vgpr4, $vgpr5
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -647,8 +647,8 @@ body: |
; GCN-LABEL: name: xcnt_event_post_load_group
; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -691,8 +691,8 @@ body: |
; GCN-LABEL: name: xcnt_event_post_store_group
; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -747,8 +747,8 @@ body: |
; GCN-LABEL: name: load_store_switching
; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr5, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: $vgpr1 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -790,8 +790,8 @@ body: |
; GCN-LABEL: name: dual_mov
; GCN: liveins: $sgpr0, $sgpr1, $vgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr2 = SCRATCH_LOAD_DWORD $vgpr1, 0, 0, implicit $exec, implicit $flat_scr
; GCN-NEXT: S_WAIT_LOADCNT 0
; GCN-NEXT: $vgpr1, $vgpr2 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
@@ -811,8 +811,8 @@ machineFunctionInfo:
body: |
bb.0:
; GCN-LABEL: name: ds_load_store
- ; GCN: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: $vgpr0 = DS_READ_B32_gfx9 killed $vgpr1, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) undef`, addrspace 3)
; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 2, implicit $exec
@@ -837,8 +837,8 @@ body: |
; GCN-LABEL: name: xcnt_max
; GCN: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
@@ -985,8 +985,8 @@ body: |
; GCN-LABEL: name: wait_kmcnt_with_outstanding_vmem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: S_WAIT_KMCNT 0
@@ -1010,8 +1010,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1052,8 +1052,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1094,8 +1094,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1140,8 +1140,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1187,8 +1187,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1236,8 +1236,8 @@ body: |
; GCN-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
; GCN-NEXT: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1, $scc
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; GCN-NEXT: {{ $}}
@@ -1282,8 +1282,8 @@ body: |
; GCN-LABEL: name: wait_loadcnt_with_outstanding_smem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: S_WAIT_LOADCNT 0
@@ -1307,8 +1307,8 @@ body: |
; GCN-LABEL: name: overwrite_vgpr_after_smem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
@@ -1328,8 +1328,8 @@ body: |
; GCN-LABEL: name: overwrite_sgpr_after_vmem
; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GCN-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GCN-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GCN-NEXT: V_NOP_e32 implicit $exec
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
; GCN-NEXT: $sgpr0 = S_MOV_B32 0
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir
index 5247bfe68226c..895daffb4b254 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir
@@ -16,8 +16,8 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $sgpr0, $vgpr0
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
; CHECK-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr28 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr32 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
index 0f2314264c30a..f44cda176449b 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir
@@ -28,8 +28,8 @@ body: |
; GFX1250-NEXT: successors: %bb.1(0x80000000)
; GFX1250-NEXT: liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; GFX1250-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_WB 0, implicit $exec
+ ; GFX1250-NEXT: V_NOP_e32 implicit $exec
; GFX1250-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
; GFX1250-NEXT: S_BRANCH %bb.1
; GFX1250-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir
index 23510a309434e..ab46a24f08d36 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir
@@ -15,8 +15,8 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $sgpr0, $vgpr0
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
; CHECK-NEXT: $vgpr10_vgpr11_vgpr12_vgpr13 = DS_READ_B128 $vgpr0, 0, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr14_vgpr15_vgpr16_vgpr17 = DS_READ_B128 $vgpr0, 16, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr18_vgpr19_vgpr20_vgpr21 = DS_READ_B128 $vgpr0, 32, 0, implicit $m0, implicit $exec
@@ -83,8 +83,8 @@ body: |
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: liveins: $sgpr0, $vgpr0
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: V_NOP_e32 implicit $exec, implicit $exec
+ ; CHECK-NEXT: GLOBAL_WB 0, implicit $exec
+ ; CHECK-NEXT: V_NOP_e32 implicit $exec
; CHECK-NEXT: $vgpr70_vgpr71_vgpr72_vgpr73 = DS_READ_B128 $vgpr0, 240, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr66_vgpr67_vgpr68_vgpr69 = DS_READ_B128 $vgpr0, 224, 0, implicit $m0, implicit $exec
; CHECK-NEXT: $vgpr62_vgpr63_vgpr64_vgpr65 = DS_READ_B128 $vgpr0, 208, 0, implicit $m0, implicit $exec
More information about the llvm-commits
mailing list