[llvm] [AMDGPU] Add GFX1250 unclaused VMEM workaround (PR #208467)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 07:01:51 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Jay Foad (jayfoad)
<details>
<summary>Changes</summary>
---
Patch is 2.90 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/208467.diff
194 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPU.td (+6)
- (modified) llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp (+13)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll (+96)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll (+22)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll (+16)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.offset-split.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll (+20)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/minmaxabs-i64.ll (+10)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll (+30)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/smul.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/add-max.ll (+44)
- (modified) llvm/test/CodeGen/AMDGPU/add_u64.ll (+16)
- (modified) llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll (+14)
- (modified) llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/bf16-conversions.ll (+28)
- (modified) llvm/test/CodeGen/AMDGPU/bf16-math.ll (+124)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+26)
- (modified) llvm/test/CodeGen/AMDGPU/bitop3.ll (+96)
- (modified) llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll (+44)
- (modified) llvm/test/CodeGen/AMDGPU/calling-conventions.ll (+110)
- (modified) llvm/test/CodeGen/AMDGPU/carryout-selection.ll (+34)
- (modified) llvm/test/CodeGen/AMDGPU/clmul.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/code-size-estimate.ll (+3-1)
- (modified) llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll (+56)
- (modified) llvm/test/CodeGen/AMDGPU/ds_write2.ll (+46)
- (modified) llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll (+138)
- (modified) llvm/test/CodeGen/AMDGPU/fcanonicalize.ll (+366)
- (modified) llvm/test/CodeGen/AMDGPU/fence-barrier-latency.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/flat-load-saddr-to-vaddr.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll (+332)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll (+464)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll (+214)
- (modified) llvm/test/CodeGen/AMDGPU/fmax3.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/fmin3.ll (+16)
- (modified) llvm/test/CodeGen/AMDGPU/fp-atomics-gfx942.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll (+98)
- (modified) llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll (+104)
- (modified) llvm/test/CodeGen/AMDGPU/global-address.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/insert-delay-alu-attr.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll (+56)
- (modified) llvm/test/CodeGen/AMDGPU/intrinsic-amdgcn-s-alloc-vgpr.ll (+18)
- (modified) llvm/test/CodeGen/AMDGPU/literal64.ll (+38)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll (+54)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.id.ll (+24)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.async.to.lds.ll (+50)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.load.ll (+28)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.id.ll (+56)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.flat.id.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cluster.workgroup.max.id.ll (+48)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cos.bf16.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll (+124)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll (+22)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.e5m3.ll (+22)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll (+132)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.ll (+40)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.f16.ll (+10)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll (+48)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll (+98)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk16.gfx1250.ll (+64)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk8.ll (+96)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk.gfx1250.ll (+96)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.sr.pk16.ll (+48)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sr.pk.bf16.ll (+10)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp.bf16.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll (+20)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.async.to.lds.ll (+30)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll (+20)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.store.async.from.lds.ll (+30)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll (+28)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll (+54)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.log.bf16.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.perm.pk.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll (+32)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll (+20)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll (+50)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll (+32)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.ll (+56)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.bf16.ll (+16)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.bf16.ll (+16)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll (+74)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.prefetch.inst.ll (+62)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.wait.gfx1250.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.gfx12.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sendmsg.rtn.ll (+26)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sin.bf16.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sqrt.bf16.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll (+34)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll (+20)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll (+36)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll (+34)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tanh.ll (+36)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll (+34)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll (+82)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1250.w32.ll (+500)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.gfx1251.w32.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1250.w32.ll (+348)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imm.gfx1251.w32.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1250.w32.ll (+392)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.imod.gfx1251.w32.ll (+20)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wmma.index.gfx1250.w32.ll (+128)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.prefetch.ll (+134)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/load-constant-i1.ll (+112)
- (modified) llvm/test/CodeGen/AMDGPU/load-constant-i32.ll (+50)
- (modified) llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/loop-prefetch-data.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/lower-work-group-id-intrinsics.ll (+16)
- (modified) llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/mad_64_32.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll (+28)
- (modified) llvm/test/CodeGen/AMDGPU/max.ll (+46)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-barriers.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll (+48)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll (+48)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll (+96)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-cluster.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll (+10)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll (+182)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll (+176)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll (+182)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-cluster.ll (+182)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll (+10)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll (+174)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-cluster.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll (+10)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-agent.ll (+182)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-cluster.ll (+182)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-lastuse.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll (+10)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-singlethread.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-system.ll (+174)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-wavefront.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/memory-legalizer-private-workgroup.ll (+184)
- (modified) llvm/test/CodeGen/AMDGPU/min.ll (+74)
- (modified) llvm/test/CodeGen/AMDGPU/minmax.ll (+40)
- (modified) llvm/test/CodeGen/AMDGPU/mul.ll (+38)
- (modified) llvm/test/CodeGen/AMDGPU/packed-fp32.ll (+156)
- (modified) llvm/test/CodeGen/AMDGPU/packed-fp64.ll (+134)
- (modified) llvm/test/CodeGen/AMDGPU/packed-u64.ll (+70)
- (modified) llvm/test/CodeGen/AMDGPU/pk-lshl-add-u64.ll (+12)
- (modified) llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll (+48)
- (modified) llvm/test/CodeGen/AMDGPU/preload-kernargs.ll (+68)
- (modified) llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll (+6)
- (modified) llvm/test/CodeGen/AMDGPU/s-barrier-signal-var-gep.ll (+26)
- (modified) llvm/test/CodeGen/AMDGPU/s-cluster-barrier.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/scale-offset-flat.ll (+48)
- (modified) llvm/test/CodeGen/AMDGPU/scale-offset-global.ll (+46)
- (modified) llvm/test/CodeGen/AMDGPU/scale-offset-scratch.ll (+44)
- (modified) llvm/test/CodeGen/AMDGPU/scale-offset-smem.ll (+42)
- (modified) llvm/test/CodeGen/AMDGPU/shl.v2i64.ll (+36)
- (modified) llvm/test/CodeGen/AMDGPU/spillv16Kernel.ll (+4)
- (modified) llvm/test/CodeGen/AMDGPU/sub_u64.ll (+18)
- (modified) llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll (+8)
- (modified) llvm/test/CodeGen/AMDGPU/wait-xcnt-atomic-rmw-optimization.ll (+40)
- (modified) llvm/test/CodeGen/AMDGPU/wait-xcnt-drain.mir (+34)
- (modified) llvm/test/CodeGen/AMDGPU/wait-xcnt.mir (+73-1)
- (modified) llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.mir (+2)
- (modified) llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-store-barrier.mir (+49-25)
- (modified) llvm/test/CodeGen/AMDGPU/waitcnt-loop-opt.mir (+4)
- (modified) llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/workgroup-id-in-arch-sgprs.ll (+12)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index a1298e5969ea0..8aed06ef918e4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1215,6 +1215,11 @@ defm FlatSignedOffset : AMDGPUSubtargetFeature<"flat-signed-offset",
"Immediate offset of FLAT instructions are always signed"
>;
+defm RequiresInitialUnclausedVmem : AMDGPUSubtargetFeature<"requires-initial-unclause-vmem",
+ "Hardware entrypoints require first VMEM instruction to be unclaused",
+ GenPredicate=false
+>;
+
//===------------------------------------------------------------===//
// Subtarget Features (options and debugging)
//===------------------------------------------------------------===//
@@ -2233,6 +2238,7 @@ def FeatureISAVersion12_50_Common : FeatureSet<
FeatureRealTrue16Insts,
FeatureVMovB64Inst,
FeatureVMulU64Inst,
+ FeatureRequiresInitialUnclausedVmem,
]>;
def FeatureISAVersion12_50 : FeatureSet<
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index c63c7c6345aa2..eb076b9a84c20 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -3629,5 +3629,18 @@ bool SIInsertWaitcnts::run() {
}
}
+ if (MFI->isEntryFunction() && ST.hasRequiresInitialUnclausedVmem()) {
+ // Hardware entrypoints must begin with a specific sequence:
+ // GLOBAL_WB SCOPE:SCOPE_CU
+ // V_NOP
+ MachineBasicBlock::iterator I = EntryBB.begin();
+ BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::GLOBAL_WB))
+ .addImm(AMDGPU::CPol::SCOPE_CU)
+ .addReg(AMDGPU::EXEC, RegState::Implicit);
+ BuildMI(EntryBB, I, DebugLoc(), TII.get(AMDGPU::V_NOP_e32))
+ .addReg(AMDGPU::EXEC, RegState::Implicit);
+ Modified = true;
+ }
+
return Modified;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index e63c31f712725..5e7d20560706d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -41,6 +41,8 @@ define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -72,6 +74,8 @@ define amdgpu_ps void @raw_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -116,6 +120,8 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -163,6 +169,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -194,6 +202,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -238,6 +248,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -285,6 +297,8 @@ define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -316,6 +330,8 @@ define amdgpu_ps void @struct_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -360,6 +376,8 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -406,6 +424,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -437,6 +457,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -481,6 +503,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -527,6 +551,8 @@ define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -558,6 +584,8 @@ define amdgpu_ps void @raw_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -602,6 +630,8 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -649,6 +679,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -680,6 +712,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -724,6 +758,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -771,6 +807,8 @@ define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -802,6 +840,8 @@ define amdgpu_ps void @struct_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -846,6 +886,8 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -892,6 +934,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -923,6 +967,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -967,6 +1013,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1013,6 +1061,8 @@ define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1044,6 +1094,8 @@ define amdgpu_ps void @raw_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, doub
;
; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1088,6 +1140,8 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr
;
; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1135,6 +1189,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8)
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1166,6 +1222,8 @@ define amdgpu_ps void @raw_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inreg
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1210,6 +1268,8 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1257,6 +1317,8 @@ define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1288,6 +1350,8 @@ define amdgpu_ps void @struct_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, d
;
; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1332,6 +1396,8 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %
;
; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1378,6 +1444,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_noret_f64(ptr addrspace(
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_noret_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1409,6 +1477,8 @@ define amdgpu_ps void @struct_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inr
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1453,6 +1523,8 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
@@ -1522,6 +1594,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -1596,6 +1670,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -1672,6 +1748,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
;
; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX1250: ; %bb.0: ; %main_body
+; GFX1250-NEXT: global_wb
+; GFX1250-NEXT: v_nop
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
@@ -1746,6 +1824,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pa...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/208467
More information about the llvm-commits
mailing list