[llvm] f2453af - [AMDGPU] Improve MemorySSA walking in isClobberedInFunction (#211817)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 07:14:54 PDT 2026
Author: Jay Foad
Date: 2026-07-29T15:14:49+01:00
New Revision: f2453af624353273325fe888d794a925ea9fe852
URL: https://github.com/llvm/llvm-project/commit/f2453af624353273325fe888d794a925ea9fe852
DIFF: https://github.com/llvm/llvm-project/commit/f2453af624353273325fe888d794a925ea9fe852.diff
LOG: [AMDGPU] Improve MemorySSA walking in isClobberedInFunction (#211817)
Consistently call the two argument overload
`MemorySSAWalker::getClobberingMemoryAccess(MemoryAccess,
MemoryLocation)` to get a clobbering def (according to the AliasAnalysis
used by MSSA) before applying the special cases in `isReallyAClobber`.
This has the effect of marking more loads as `amdgpu-noclobber` and
selecting SMEM load instructions for them.
Fixes: ROCM-28492
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
llvm/test/CodeGen/AMDGPU/global_smrd_cfg.ll
llvm/test/CodeGen/AMDGPU/move-to-valu-addsubu64.ll
llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index ae0a614bddbf2..781d591118604 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -409,9 +409,11 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {
bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
AAResults *AA) {
MemorySSAWalker *Walker = MSSA->getWalker();
- SmallVector<MemoryAccess *> WorkList{Walker->getClobberingMemoryAccess(Load)};
- SmallPtrSet<MemoryAccess *, 8> Visited;
MemoryLocation Loc(MemoryLocation::get(Load));
+ MemoryUseOrDef *Use = MSSA->getMemoryAccess(Load);
+ SmallVector<MemoryAccess *> WorkList{
+ Walker->getClobberingMemoryAccess(Use->getDefiningAccess(), Loc)};
+ SmallPtrSet<MemoryAccess *, 8> Visited;
LLVM_DEBUG(dbgs() << "Checking clobbering of: " << *Load << '\n');
@@ -446,7 +448,8 @@ bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
const MemoryPhi *Phi = cast<MemoryPhi>(MA);
for (const auto &Use : Phi->incoming_values())
- WorkList.push_back(cast<MemoryAccess>(&Use));
+ WorkList.push_back(
+ Walker->getClobberingMemoryAccess(cast<MemoryAccess>(&Use), Loc));
}
LLVM_DEBUG(dbgs() << " -> no clobber\n");
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 960aa6932c9ed..41c3e0a07f5f5 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -510,346 +510,338 @@ define void @v32_asm_def_use(float %v0, float %v1) #4 {
define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg2, i64 %arg3, <2 x half> %arg4, <2 x half> %arg5) #3 {
; GFX908-LABEL: introduced_copy_to_sgpr:
; GFX908: ; %bb.0: ; %bb
-; GFX908-NEXT: global_load_ushort v16, v[0:1], off glc
-; GFX908-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
-; GFX908-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x10
-; GFX908-NEXT: s_load_dword s0, s[8:9], 0x18
-; GFX908-NEXT: s_mov_b32 s10, 0
-; GFX908-NEXT: s_mov_b32 s9, s10
+; GFX908-NEXT: global_load_ushort v0, v[0:1], off glc
+; GFX908-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX908-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX908-NEXT: s_load_dword s7, s[8:9], 0x18
+; GFX908-NEXT: s_mov_b32 s6, 0
+; GFX908-NEXT: s_mov_b32 s9, s6
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX908-NEXT: s_sub_i32 s1, 0, s7
-; GFX908-NEXT: v_cvt_f32_f16_e32 v18, s0
-; GFX908-NEXT: v_mov_b32_e32 v17, 0
-; GFX908-NEXT: v_rcp_f32_e32 v0, v0
-; GFX908-NEXT: v_mov_b32_e32 v1, 0
-; GFX908-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX908-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX908-NEXT: v_readfirstlane_b32 s8, v0
-; GFX908-NEXT: s_mul_i32 s1, s1, s8
-; GFX908-NEXT: s_mul_hi_u32 s1, s8, s1
-; GFX908-NEXT: s_add_i32 s8, s8, s1
-; GFX908-NEXT: s_mul_hi_u32 s1, s6, s8
-; GFX908-NEXT: s_mul_i32 s8, s1, s7
-; GFX908-NEXT: s_sub_i32 s6, s6, s8
-; GFX908-NEXT: s_add_i32 s11, s1, 1
-; GFX908-NEXT: s_sub_i32 s8, s6, s7
-; GFX908-NEXT: s_cmp_ge_u32 s6, s7
-; GFX908-NEXT: s_cselect_b32 s1, s11, s1
-; GFX908-NEXT: s_cselect_b32 s6, s8, s6
-; GFX908-NEXT: s_add_i32 s8, s1, 1
-; GFX908-NEXT: s_cmp_ge_u32 s6, s7
-; GFX908-NEXT: s_cselect_b32 s8, s8, s1
-; GFX908-NEXT: s_lshr_b32 s11, s0, 16
-; GFX908-NEXT: s_lshl_b64 s[14:15], s[8:9], 5
-; GFX908-NEXT: v_cvt_f32_f16_e32 v19, s11
-; GFX908-NEXT: s_lshl_b64 s[6:7], s[4:5], 5
-; GFX908-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
-; GFX908-NEXT: s_and_b64 s[0:1], exec, s[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v0, 0
+; GFX908-NEXT: v_cvt_f32_u32_e32 v1, s3
+; GFX908-NEXT: s_sub_i32 s8, 0, s3
+; GFX908-NEXT: v_cvt_f32_f16_e32 v12, s7
+; GFX908-NEXT: s_mov_b64 s[16:17], 0
+; GFX908-NEXT: v_rcp_f32_e32 v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v14, 0
+; GFX908-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
+; GFX908-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX908-NEXT: v_readfirstlane_b32 s10, v1
+; GFX908-NEXT: s_mul_i32 s8, s8, s10
+; GFX908-NEXT: s_mul_hi_u32 s8, s10, s8
+; GFX908-NEXT: s_add_i32 s10, s10, s8
+; GFX908-NEXT: s_mul_hi_u32 s8, s2, s10
+; GFX908-NEXT: s_mul_i32 s10, s8, s3
+; GFX908-NEXT: s_sub_i32 s2, s2, s10
+; GFX908-NEXT: s_add_i32 s11, s8, 1
+; GFX908-NEXT: s_sub_i32 s10, s2, s3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s9, v16
-; GFX908-NEXT: s_and_b32 s9, 0xffff, s9
-; GFX908-NEXT: s_mul_i32 s5, s5, s9
-; GFX908-NEXT: s_mul_hi_u32 s11, s4, s9
-; GFX908-NEXT: s_mul_i32 s4, s4, s9
-; GFX908-NEXT: s_add_i32 s5, s11, s5
-; GFX908-NEXT: s_lshl_b64 s[4:5], s[4:5], 5
+; GFX908-NEXT: v_readfirstlane_b32 s12, v0
+; GFX908-NEXT: s_and_b32 s30, s12, 0xffff
+; GFX908-NEXT: s_cmp_ge_u32 s2, s3
+; GFX908-NEXT: s_cselect_b32 s8, s11, s8
+; GFX908-NEXT: s_cselect_b32 s2, s10, s2
+; GFX908-NEXT: s_add_i32 s10, s8, 1
+; GFX908-NEXT: s_cmp_ge_u32 s2, s3
+; GFX908-NEXT: s_cselect_b32 s8, s10, s8
+; GFX908-NEXT: s_lshr_b32 s7, s7, 16
+; GFX908-NEXT: v_cvt_f32_f16_e32 v13, s7
+; GFX908-NEXT: s_mul_i32 s12, s1, s30
+; GFX908-NEXT: s_mul_hi_u32 s13, s0, s30
+; GFX908-NEXT: s_mul_i32 s14, s0, s30
+; GFX908-NEXT: s_add_i32 s15, s13, s12
+; GFX908-NEXT: s_lshl_b64 s[2:3], s[0:1], 5
+; GFX908-NEXT: s_lshl_b64 s[10:11], s[4:5], 5
+; GFX908-NEXT: s_lshl_b64 s[12:13], s[8:9], 5
+; GFX908-NEXT: s_lshl_b64 s[14:15], s[14:15], 5
+; GFX908-NEXT: s_and_b64 s[0:1], exec, s[0:1]
; GFX908-NEXT: s_branch .LBB3_2
; GFX908-NEXT: .LBB3_1: ; %Flow20
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_and_b64 s[16:17], s[16:17], exec
-; GFX908-NEXT: s_cselect_b32 s9, 1, 0
-; GFX908-NEXT: s_cmp_lg_u32 s9, 1
-; GFX908-NEXT: s_cbranch_scc0 .LBB3_14
+; GFX908-NEXT: s_and_b64 s[18:19], s[18:19], exec
+; GFX908-NEXT: s_cselect_b32 s7, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s7, 1
+; GFX908-NEXT: s_cbranch_scc0 .LBB3_13
; GFX908-NEXT: .LBB3_2: ; %bb9
; GFX908-NEXT: ; =>This Loop Header: Depth=1
-; GFX908-NEXT: ; Child Loop BB3_6 Depth 2
-; GFX908-NEXT: s_mov_b64 s[18:19], -1
+; GFX908-NEXT: ; Child Loop BB3_5 Depth 2
+; GFX908-NEXT: s_mov_b64 s[20:21], -1
; GFX908-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX908-NEXT: s_cbranch_vccz .LBB3_12
+; GFX908-NEXT: s_cbranch_vccz .LBB3_11
; GFX908-NEXT: ; %bb.3: ; %bb14
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
-; GFX908-NEXT: s_cmp_lt_i32 s3, 0
-; GFX908-NEXT: s_mov_b32 s11, s10
-; GFX908-NEXT: s_cselect_b64 s[16:17], -1, 0
-; GFX908-NEXT: s_cmp_gt_i32 s3, -1
-; GFX908-NEXT: v_mov_b32_e32 v4, s10
-; GFX908-NEXT: v_mov_b32_e32 v5, s11
+; GFX908-NEXT: s_load_dwordx2 s[18:19], s[16:17], 0x0
+; GFX908-NEXT: s_cmp_lt_i32 s5, 0
; GFX908-NEXT: s_cselect_b64 s[20:21], -1, 0
-; GFX908-NEXT: v_mov_b32_e32 v6, s10
-; GFX908-NEXT: v_mov_b32_e32 v7, s11
-; GFX908-NEXT: v_mov_b32_e32 v8, s10
-; GFX908-NEXT: v_mov_b32_e32 v9, s11
-; GFX908-NEXT: s_mov_b64 s[18:19], s[12:13]
-; GFX908-NEXT: v_mov_b32_e32 v11, v5
-; GFX908-NEXT: v_mov_b32_e32 v10, v4
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s9, v2
-; GFX908-NEXT: v_readfirstlane_b32 s11, v3
-; GFX908-NEXT: s_add_u32 s9, s9, 1
-; GFX908-NEXT: s_addc_u32 s11, s11, 0
-; GFX908-NEXT: s_mul_hi_u32 s22, s6, s9
-; GFX908-NEXT: s_mul_i32 s11, s6, s11
-; GFX908-NEXT: s_mul_i32 s23, s7, s9
-; GFX908-NEXT: s_add_i32 s11, s22, s11
-; GFX908-NEXT: s_mul_i32 s9, s6, s9
-; GFX908-NEXT: s_add_i32 s11, s11, s23
-; GFX908-NEXT: s_branch .LBB3_6
-; GFX908-NEXT: .LBB3_4: ; %bb58
-; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX908-NEXT: v_add_co_u32_sdwa v2, vcc, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX908-NEXT: s_add_u32 s18, s18, s4
-; GFX908-NEXT: s_addc_u32 s19, s19, s5
-; GFX908-NEXT: s_mov_b64 s[22:23], 0
-; GFX908-NEXT: v_cmp_lt_i32_e64 s[24:25], -1, v3
-; GFX908-NEXT: .LBB3_5: ; %Flow18
-; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX908-NEXT: s_and_b64 s[24:25], s[24:25], exec
-; GFX908-NEXT: s_cselect_b32 s24, 1, 0
-; GFX908-NEXT: s_cmp_lg_u32 s24, 1
-; GFX908-NEXT: s_cbranch_scc0 .LBB3_11
-; GFX908-NEXT: .LBB3_6: ; %bb16
+; GFX908-NEXT: s_cmp_gt_i32 s5, -1
+; GFX908-NEXT: s_cselect_b64 s[22:23], -1, 0
+; GFX908-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-NEXT: s_add_u32 s7, s18, 1
+; GFX908-NEXT: s_addc_u32 s9, s19, 0
+; GFX908-NEXT: s_mul_i32 s9, s2, s9
+; GFX908-NEXT: s_mul_hi_u32 s24, s2, s7
+; GFX908-NEXT: s_add_i32 s9, s24, s9
+; GFX908-NEXT: s_mul_i32 s24, s3, s7
+; GFX908-NEXT: s_mul_i32 s31, s2, s7
+; GFX908-NEXT: s_mov_b32 s7, s6
+; GFX908-NEXT: s_add_i32 s9, s9, s24
+; GFX908-NEXT: v_mov_b32_e32 v0, s6
+; GFX908-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-NEXT: s_mov_b64 s[24:25], s[10:11]
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: v_mov_b32_e32 v3, s7
+; GFX908-NEXT: v_mov_b32_e32 v4, s6
+; GFX908-NEXT: v_mov_b32_e32 v5, s7
+; GFX908-NEXT: v_mov_b32_e32 v6, s6
+; GFX908-NEXT: v_mov_b32_e32 v7, s7
+; GFX908-NEXT: s_branch .LBB3_5
+; GFX908-NEXT: .LBB3_4: ; %Flow18
+; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX908-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; GFX908-NEXT: s_cselect_b32 s7, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s7, 1
+; GFX908-NEXT: s_cbranch_scc0 .LBB3_10
+; GFX908-NEXT: .LBB3_5: ; %bb16
; GFX908-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT: s_add_u32 s22, s18, s9
-; GFX908-NEXT: s_addc_u32 s23, s19, s11
-; GFX908-NEXT: global_load_dword v21, v17, s[22:23] offset:16 glc
+; GFX908-NEXT: s_add_u32 s26, s24, s31
+; GFX908-NEXT: s_addc_u32 s27, s25, s9
+; GFX908-NEXT: global_load_dword v16, v14, s[26:27] offset:16 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_load_dword v20, v17, s[22:23] offset:20 glc
+; GFX908-NEXT: global_load_dword v15, v14, s[26:27] offset:20 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_load_dword v12, v17, s[22:23] offset:24 glc
+; GFX908-NEXT: global_load_dword v8, v14, s[26:27] offset:24 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: global_load_dword v12, v17, s[22:23] offset:28 glc
+; GFX908-NEXT: global_load_dword v8, v14, s[26:27] offset:28 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: ds_read_b64 v[12:13], v17
-; GFX908-NEXT: ds_read_b64 v[14:15], v0
-; GFX908-NEXT: s_and_b64 s[22:23], s[20:21], exec
-; GFX908-NEXT: s_cselect_b32 s22, 1, 0
-; GFX908-NEXT: s_cmp_lg_u32 s22, 1
+; GFX908-NEXT: ds_read_b64 v[8:9], v14
+; GFX908-NEXT: ds_read_b64 v[10:11], v0
+; GFX908-NEXT: ; kill: killed $sgpr26 killed $sgpr27
+; GFX908-NEXT: s_and_b64 s[26:27], s[22:23], exec
+; GFX908-NEXT: s_cselect_b32 s7, 1, 0
+; GFX908-NEXT: s_cmp_lg_u32 s7, 1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_cbranch_scc1 .LBB3_8
-; GFX908-NEXT: ; %bb.7: ; %bb51
-; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX908-NEXT: v_add_f32_e32 v22, v18, v12
-; GFX908-NEXT: v_add_f32_e32 v23, v19, v13
-; GFX908-NEXT: v_add_f32_e32 v24, 0, v12
-; GFX908-NEXT: v_add_f32_e32 v25, 0, v13
-; GFX908-NEXT: v_fma_mix_f32 v14, v21, 1.0, v14 op_sel_hi:[1,1,0]
-; GFX908-NEXT: v_fma_mix_f32 v15, v21, 1.0, v15 op_sel:[1,0,0] op_sel_hi:[1,1,0]
-; GFX908-NEXT: v_fma_mix_f32 v12, v20, 1.0, v12 op_sel_hi:[1,1,0]
-; GFX908-NEXT: v_fma_mix_f32 v13, v20, 1.0, v13 op_sel:[1,0,0] op_sel_hi:[1,1,0]
-; GFX908-NEXT: v_add_f32_e32 v5, v5, v23
-; GFX908-NEXT: v_add_f32_e32 v4, v4, v22
-; GFX908-NEXT: v_add_f32_e32 v7, v7, v25
-; GFX908-NEXT: v_add_f32_e32 v6, v6, v24
-; GFX908-NEXT: v_add_f32_e32 v9, v9, v15
-; GFX908-NEXT: v_add_f32_e32 v8, v8, v14
-; GFX908-NEXT: v_add_f32_e32 v11, v11, v13
-; GFX908-NEXT: v_add_f32_e32 v10, v10, v12
-; GFX908-NEXT: s_mov_b64 s[22:23], -1
-; GFX908-NEXT: s_branch .LBB3_9
-; GFX908-NEXT: .LBB3_8: ; in Loop: Header=BB3_6 Depth=2
-; GFX908-NEXT: s_mov_b64 s[22:23], s[16:17]
-; GFX908-NEXT: .LBB3_9: ; %Flow
-; GFX908-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX908-NEXT: s_and_b64 s[22:23], s[22:23], exec
-; GFX908-NEXT: s_cselect_b32 s22, 1, 0
-; GFX908-NEXT: s_cmp_lg_u32 s22, 1
-; GFX908-NEXT: s_cbranch_scc0 .LBB3_4
-; GFX908-NEXT: ; %bb.10: ; in Loop: Header=BB3_6 Depth=2
-; GFX908-NEXT: s_mov_b64 s[22:23], -1
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX908-NEXT: ; implicit-def: $sgpr18_sgpr19
-; GFX908-NEXT: s_mov_b64 s[24:25], -1
-; GFX908-NEXT: s_branch .LBB3_5
-; GFX908-NEXT: .LBB3_11: ; %loop.exit.guard
+; GFX908-NEXT: s_cbranch_scc1 .LBB3_7
+; GFX908-NEXT: ; %bb.6: ; %bb51
+; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX908-NEXT: v_add_f32_e32 v17, v12, v8
+; GFX908-NEXT: v_add_f32_e32 v18, v13, v9
+; GFX908-NEXT: v_add_f32_e32 v19, 0, v8
+; GFX908-NEXT: v_add_f32_e32 v20, 0, v9
+; GFX908-NEXT: v_fma_mix_f32 v10, v16, 1.0, v10 op_sel_hi:[1,1,0]
+; GFX908-NEXT: v_fma_mix_f32 v11, v16, 1.0, v11 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; GFX908-NEXT: v_fma_mix_f32 v8, v15, 1.0, v8 op_sel_hi:[1,1,0]
+; GFX908-NEXT: v_fma_mix_f32 v9, v15, 1.0, v9 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; GFX908-NEXT: v_add_f32_e32 v1, v1, v18
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v17
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v20
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v19
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v11
+; GFX908-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX908-NEXT: v_add_f32_e32 v7, v7, v9
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v8
+; GFX908-NEXT: s_mov_b64 s[28:29], -1
+; GFX908-NEXT: s_branch .LBB3_8
+; GFX908-NEXT: .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
+; GFX908-NEXT: s_mov_b64 s[28:29], s[20:21]
+; GFX908-NEXT: .LBB3_8: ; %Flow
+; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX908-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; GFX908-NEXT: s_cselect_b32 s7, 1, 0
+; GFX908-NEXT: s_mov_b64 s[26:27], -1
+; GFX908-NEXT: s_cmp_lg_u32 s7, 1
+; GFX908-NEXT: s_mov_b64 s[28:29], -1
+; GFX908-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX908-NEXT: ; %bb.9: ; %bb58
+; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX908-NEXT: s_add_u32 s18, s18, s30
+; GFX908-NEXT: s_addc_u32 s19, s19, 0
+; GFX908-NEXT: s_add_u32 s24, s24, s14
+; GFX908-NEXT: s_addc_u32 s25, s25, s15
+; GFX908-NEXT: s_cmp_gt_i32 s19, -1
+; GFX908-NEXT: s_mov_b64 s[26:27], 0
+; GFX908-NEXT: s_cselect_b64 s[28:29], -1, 0
+; GFX908-NEXT: s_branch .LBB3_4
+; GFX908-NEXT: .LBB3_10: ; %loop.exit.guard
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_xor_b64 s[18:19], s[22:23], -1
-; GFX908-NEXT: .LBB3_12: ; %Flow19
+; GFX908-NEXT: s_xor_b64 s[20:21], s[26:27], -1
+; GFX908-NEXT: .LBB3_11: ; %Flow19
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_mov_b64 s[16:17], -1
-; GFX908-NEXT: s_and_b64 vcc, exec, s[18:19]
+; GFX908-NEXT: s_mov_b64 s[18:19], -1
+; GFX908-NEXT: s_and_b64 vcc, exec, s[20:21]
; GFX908-NEXT: s_cbranch_vccz .LBB3_1
-; GFX908-NEXT: ; %bb.13: ; %bb12
+; GFX908-NEXT: ; %bb.12: ; %bb12
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_add_u32 s2, s2, s8
-; GFX908-NEXT: s_addc_u32 s3, s3, 0
-; GFX908-NEXT: s_add_u32 s12, s12, s14
-; GFX908-NEXT: s_addc_u32 s13, s13, s15
-; GFX908-NEXT: s_mov_b64 s[16:17], 0
+; GFX908-NEXT: s_add_u32 s4, s4, s8
+; GFX908-NEXT: s_addc_u32 s5, s5, 0
+; GFX908-NEXT: s_add_u32 s10, s10, s12
+; GFX908-NEXT: s_addc_u32 s11, s11, s13
+; GFX908-NEXT: s_mov_b64 s[18:19], 0
; GFX908-NEXT: s_branch .LBB3_1
-; GFX908-NEXT: .LBB3_14: ; %DummyReturnBlock
+; GFX908-NEXT: .LBB3_13: ; %DummyReturnBlock
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: introduced_copy_to_sgpr:
; GFX90A: ; %bb.0: ; %bb
-; GFX90A-NEXT: global_load_ushort v18, v[0:1], off glc
-; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
-; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x10
-; GFX90A-NEXT: s_load_dword s0, s[8:9], 0x18
-; GFX90A-NEXT: s_mov_b32 s10, 0
-; GFX90A-NEXT: s_mov_b32 s9, s10
+; GFX90A-NEXT: global_load_ushort v1, v[0:1], off glc
+; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX90A-NEXT: s_load_dword s7, s[8:9], 0x18
+; GFX90A-NEXT: s_mov_b32 s6, 0
+; GFX90A-NEXT: s_mov_b32 s9, s6
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX90A-NEXT: s_sub_i32 s1, 0, s7
-; GFX90A-NEXT: v_mov_b32_e32 v19, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], 0, 0
+; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX90A-NEXT: s_sub_i32 s8, 0, s3
+; GFX90A-NEXT: s_mov_b64 s[16:17], 0
+; GFX90A-NEXT: v_mov_b32_e32 v14, 0
; GFX90A-NEXT: v_rcp_f32_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX90A-NEXT: v_cvt_u32_f32_e32 v1, v0
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
-; GFX90A-NEXT: s_mul_i32 s1, s1, s8
-; GFX90A-NEXT: s_mul_hi_u32 s1, s8, s1
-; GFX90A-NEXT: s_add_i32 s8, s8, s1
-; GFX90A-NEXT: s_mul_hi_u32 s1, s6, s8
-; GFX90A-NEXT: s_mul_i32 s8, s1, s7
-; GFX90A-NEXT: s_sub_i32 s6, s6, s8
-; GFX90A-NEXT: s_add_i32 s11, s1, 1
-; GFX90A-NEXT: s_sub_i32 s8, s6, s7
-; GFX90A-NEXT: s_cmp_ge_u32 s6, s7
-; GFX90A-NEXT: s_cselect_b32 s1, s11, s1
-; GFX90A-NEXT: s_cselect_b32 s6, s8, s6
-; GFX90A-NEXT: s_add_i32 s8, s1, 1
-; GFX90A-NEXT: s_cmp_ge_u32 s6, s7
-; GFX90A-NEXT: s_cselect_b32 s8, s8, s1
-; GFX90A-NEXT: s_lshr_b32 s11, s0, 16
-; GFX90A-NEXT: s_lshl_b64 s[14:15], s[8:9], 5
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v1, s11
-; GFX90A-NEXT: s_lshl_b64 s[6:7], s[4:5], 5
-; GFX90A-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
-; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[0:1]
+; GFX90A-NEXT: v_cvt_u32_f32_e32 v2, v0
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v0, s7
+; GFX90A-NEXT: v_readfirstlane_b32 s10, v2
+; GFX90A-NEXT: s_mul_i32 s8, s8, s10
+; GFX90A-NEXT: s_mul_hi_u32 s8, s10, s8
+; GFX90A-NEXT: s_add_i32 s10, s10, s8
+; GFX90A-NEXT: s_mul_hi_u32 s8, s2, s10
+; GFX90A-NEXT: s_mul_i32 s10, s8, s3
+; GFX90A-NEXT: s_sub_i32 s2, s2, s10
+; GFX90A-NEXT: s_add_i32 s11, s8, 1
+; GFX90A-NEXT: s_sub_i32 s10, s2, s3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s9, v18
-; GFX90A-NEXT: s_and_b32 s9, 0xffff, s9
-; GFX90A-NEXT: s_mul_i32 s5, s5, s9
-; GFX90A-NEXT: s_mul_hi_u32 s11, s4, s9
-; GFX90A-NEXT: s_mul_i32 s4, s4, s9
-; GFX90A-NEXT: s_add_i32 s5, s11, s5
-; GFX90A-NEXT: s_lshl_b64 s[4:5], s[4:5], 5
+; GFX90A-NEXT: v_readfirstlane_b32 s12, v1
+; GFX90A-NEXT: s_and_b32 s30, s12, 0xffff
+; GFX90A-NEXT: s_cmp_ge_u32 s2, s3
+; GFX90A-NEXT: s_cselect_b32 s8, s11, s8
+; GFX90A-NEXT: s_cselect_b32 s2, s10, s2
+; GFX90A-NEXT: s_add_i32 s10, s8, 1
+; GFX90A-NEXT: s_cmp_ge_u32 s2, s3
+; GFX90A-NEXT: s_cselect_b32 s8, s10, s8
+; GFX90A-NEXT: s_lshr_b32 s7, s7, 16
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v1, s7
+; GFX90A-NEXT: s_mul_i32 s12, s1, s30
+; GFX90A-NEXT: s_mul_hi_u32 s13, s0, s30
+; GFX90A-NEXT: s_mul_i32 s14, s0, s30
+; GFX90A-NEXT: s_add_i32 s15, s13, s12
+; GFX90A-NEXT: s_lshl_b64 s[2:3], s[0:1], 5
+; GFX90A-NEXT: s_lshl_b64 s[10:11], s[4:5], 5
+; GFX90A-NEXT: s_lshl_b64 s[12:13], s[8:9], 5
+; GFX90A-NEXT: s_lshl_b64 s[14:15], s[14:15], 5
+; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[0:1]
; GFX90A-NEXT: s_branch .LBB3_2
; GFX90A-NEXT: .LBB3_1: ; %Flow20
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_and_b64 s[16:17], s[16:17], exec
-; GFX90A-NEXT: s_cselect_b32 s9, 1, 0
-; GFX90A-NEXT: s_cmp_lg_u32 s9, 1
-; GFX90A-NEXT: s_cbranch_scc0 .LBB3_14
+; GFX90A-NEXT: s_and_b64 s[18:19], s[18:19], exec
+; GFX90A-NEXT: s_cselect_b32 s7, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s7, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB3_13
; GFX90A-NEXT: .LBB3_2: ; %bb9
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
-; GFX90A-NEXT: ; Child Loop BB3_6 Depth 2
-; GFX90A-NEXT: s_mov_b64 s[18:19], -1
+; GFX90A-NEXT: ; Child Loop BB3_5 Depth 2
+; GFX90A-NEXT: s_mov_b64 s[20:21], -1
; GFX90A-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX90A-NEXT: s_cbranch_vccz .LBB3_12
+; GFX90A-NEXT: s_cbranch_vccz .LBB3_11
; GFX90A-NEXT: ; %bb.3: ; %bb14
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX90A-NEXT: s_cmp_lt_i32 s3, 0
-; GFX90A-NEXT: s_mov_b32 s11, s10
-; GFX90A-NEXT: s_cselect_b64 s[16:17], -1, 0
-; GFX90A-NEXT: s_cmp_gt_i32 s3, -1
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[10:11], s[10:11] op_sel:[0,1]
+; GFX90A-NEXT: s_load_dwordx2 s[18:19], s[16:17], 0x0
+; GFX90A-NEXT: s_cmp_lt_i32 s5, 0
; GFX90A-NEXT: s_cselect_b64 s[20:21], -1, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
-; GFX90A-NEXT: s_mov_b64 s[18:19], s[12:13]
-; GFX90A-NEXT: v_pk_mov_b32 v[12:13], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s9, v4
-; GFX90A-NEXT: v_readfirstlane_b32 s11, v5
-; GFX90A-NEXT: s_add_u32 s9, s9, 1
-; GFX90A-NEXT: s_addc_u32 s11, s11, 0
-; GFX90A-NEXT: s_mul_hi_u32 s22, s6, s9
-; GFX90A-NEXT: s_mul_i32 s11, s6, s11
-; GFX90A-NEXT: s_mul_i32 s23, s7, s9
-; GFX90A-NEXT: s_add_i32 s11, s22, s11
-; GFX90A-NEXT: s_mul_i32 s9, s6, s9
-; GFX90A-NEXT: s_add_i32 s11, s11, s23
-; GFX90A-NEXT: s_branch .LBB3_6
-; GFX90A-NEXT: .LBB3_4: ; %bb58
-; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX90A-NEXT: v_add_co_u32_sdwa v4, vcc, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX90A-NEXT: s_add_u32 s18, s18, s4
-; GFX90A-NEXT: s_addc_u32 s19, s19, s5
-; GFX90A-NEXT: s_mov_b64 s[22:23], 0
-; GFX90A-NEXT: v_cmp_lt_i32_e64 s[24:25], -1, v5
-; GFX90A-NEXT: .LBB3_5: ; %Flow18
-; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX90A-NEXT: s_and_b64 s[24:25], s[24:25], exec
-; GFX90A-NEXT: s_cselect_b32 s24, 1, 0
-; GFX90A-NEXT: s_cmp_lg_u32 s24, 1
-; GFX90A-NEXT: s_cbranch_scc0 .LBB3_11
-; GFX90A-NEXT: .LBB3_6: ; %bb16
+; GFX90A-NEXT: s_cmp_gt_i32 s5, -1
+; GFX90A-NEXT: s_cselect_b64 s[22:23], -1, 0
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_add_u32 s7, s18, 1
+; GFX90A-NEXT: s_addc_u32 s9, s19, 0
+; GFX90A-NEXT: s_mul_i32 s9, s2, s9
+; GFX90A-NEXT: s_mul_hi_u32 s24, s2, s7
+; GFX90A-NEXT: s_add_i32 s9, s24, s9
+; GFX90A-NEXT: s_mul_i32 s24, s3, s7
+; GFX90A-NEXT: s_mul_i32 s31, s2, s7
+; GFX90A-NEXT: s_mov_b32 s7, s6
+; GFX90A-NEXT: s_add_i32 s9, s9, s24
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-NEXT: s_mov_b64 s[24:25], s[10:11]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[6:7], s[6:7] op_sel:[0,1]
+; GFX90A-NEXT: s_branch .LBB3_5
+; GFX90A-NEXT: .LBB3_4: ; %Flow18
+; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX90A-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; GFX90A-NEXT: s_cselect_b32 s7, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s7, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB3_10
+; GFX90A-NEXT: .LBB3_5: ; %bb16
; GFX90A-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX90A-NEXT: s_add_u32 s22, s18, s9
-; GFX90A-NEXT: s_addc_u32 s23, s19, s11
-; GFX90A-NEXT: global_load_dword v21, v19, s[22:23] offset:16 glc
+; GFX90A-NEXT: s_add_u32 s26, s24, s31
+; GFX90A-NEXT: s_addc_u32 s27, s25, s9
+; GFX90A-NEXT: global_load_dword v16, v14, s[26:27] offset:16 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_load_dword v20, v19, s[22:23] offset:20 glc
+; GFX90A-NEXT: global_load_dword v15, v14, s[26:27] offset:20 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_load_dword v14, v19, s[22:23] offset:24 glc
+; GFX90A-NEXT: global_load_dword v10, v14, s[26:27] offset:24 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_load_dword v14, v19, s[22:23] offset:28 glc
+; GFX90A-NEXT: global_load_dword v10, v14, s[26:27] offset:28 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: ds_read_b64 v[14:15], v19
-; GFX90A-NEXT: ds_read_b64 v[16:17], v0
-; GFX90A-NEXT: ; kill: killed $sgpr22 killed $sgpr23
-; GFX90A-NEXT: s_and_b64 s[22:23], s[20:21], exec
-; GFX90A-NEXT: s_cselect_b32 s22, 1, 0
-; GFX90A-NEXT: s_cmp_lg_u32 s22, 1
+; GFX90A-NEXT: ds_read_b64 v[10:11], v14
+; GFX90A-NEXT: ds_read_b64 v[12:13], v0
+; GFX90A-NEXT: ; kill: killed $sgpr26 killed $sgpr27
+; GFX90A-NEXT: s_and_b64 s[26:27], s[22:23], exec
+; GFX90A-NEXT: s_cselect_b32 s7, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s7, 1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_cbranch_scc1 .LBB3_8
-; GFX90A-NEXT: ; %bb.7: ; %bb51
-; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX90A-NEXT: v_cvt_f32_f16_sdwa v23, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v22, v21
-; GFX90A-NEXT: v_cvt_f32_f16_sdwa v21, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v20, v20
-; GFX90A-NEXT: v_pk_add_f32 v[24:25], v[0:1], v[14:15]
-; GFX90A-NEXT: v_pk_add_f32 v[26:27], v[14:15], 0 op_sel_hi:[1,0]
-; GFX90A-NEXT: v_pk_add_f32 v[16:17], v[22:23], v[16:17]
-; GFX90A-NEXT: v_pk_add_f32 v[14:15], v[20:21], v[14:15]
-; GFX90A-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[24:25]
-; GFX90A-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[26:27]
-; GFX90A-NEXT: v_pk_add_f32 v[10:11], v[10:11], v[16:17]
-; GFX90A-NEXT: v_pk_add_f32 v[12:13], v[12:13], v[14:15]
-; GFX90A-NEXT: s_mov_b64 s[22:23], -1
-; GFX90A-NEXT: s_branch .LBB3_9
-; GFX90A-NEXT: .LBB3_8: ; in Loop: Header=BB3_6 Depth=2
-; GFX90A-NEXT: s_mov_b64 s[22:23], s[16:17]
-; GFX90A-NEXT: .LBB3_9: ; %Flow
-; GFX90A-NEXT: ; in Loop: Header=BB3_6 Depth=2
-; GFX90A-NEXT: s_and_b64 s[22:23], s[22:23], exec
-; GFX90A-NEXT: s_cselect_b32 s22, 1, 0
-; GFX90A-NEXT: s_cmp_lg_u32 s22, 1
-; GFX90A-NEXT: s_cbranch_scc0 .LBB3_4
-; GFX90A-NEXT: ; %bb.10: ; in Loop: Header=BB3_6 Depth=2
-; GFX90A-NEXT: s_mov_b64 s[22:23], -1
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX90A-NEXT: ; implicit-def: $sgpr18_sgpr19
-; GFX90A-NEXT: s_mov_b64 s[24:25], -1
-; GFX90A-NEXT: s_branch .LBB3_5
-; GFX90A-NEXT: .LBB3_11: ; %loop.exit.guard
+; GFX90A-NEXT: s_cbranch_scc1 .LBB3_7
+; GFX90A-NEXT: ; %bb.6: ; %bb51
+; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX90A-NEXT: v_cvt_f32_f16_sdwa v17, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v16, v16
+; GFX90A-NEXT: v_cvt_f32_f16_sdwa v19, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v18, v15
+; GFX90A-NEXT: v_pk_add_f32 v[20:21], v[0:1], v[10:11]
+; GFX90A-NEXT: v_pk_add_f32 v[22:23], v[10:11], 0 op_sel_hi:[1,0]
+; GFX90A-NEXT: v_pk_add_f32 v[12:13], v[16:17], v[12:13]
+; GFX90A-NEXT: v_pk_add_f32 v[10:11], v[18:19], v[10:11]
+; GFX90A-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[20:21]
+; GFX90A-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[22:23]
+; GFX90A-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[12:13]
+; GFX90A-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[10:11]
+; GFX90A-NEXT: s_mov_b64 s[28:29], -1
+; GFX90A-NEXT: s_branch .LBB3_8
+; GFX90A-NEXT: .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
+; GFX90A-NEXT: s_mov_b64 s[28:29], s[20:21]
+; GFX90A-NEXT: .LBB3_8: ; %Flow
+; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX90A-NEXT: s_and_b64 s[28:29], s[28:29], exec
+; GFX90A-NEXT: s_cselect_b32 s7, 1, 0
+; GFX90A-NEXT: s_mov_b64 s[26:27], -1
+; GFX90A-NEXT: s_cmp_lg_u32 s7, 1
+; GFX90A-NEXT: s_mov_b64 s[28:29], -1
+; GFX90A-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX90A-NEXT: ; %bb.9: ; %bb58
+; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
+; GFX90A-NEXT: s_add_u32 s18, s18, s30
+; GFX90A-NEXT: s_addc_u32 s19, s19, 0
+; GFX90A-NEXT: s_add_u32 s24, s24, s14
+; GFX90A-NEXT: s_addc_u32 s25, s25, s15
+; GFX90A-NEXT: s_cmp_gt_i32 s19, -1
+; GFX90A-NEXT: s_mov_b64 s[26:27], 0
+; GFX90A-NEXT: s_cselect_b64 s[28:29], -1, 0
+; GFX90A-NEXT: s_branch .LBB3_4
+; GFX90A-NEXT: .LBB3_10: ; %loop.exit.guard
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_xor_b64 s[18:19], s[22:23], -1
-; GFX90A-NEXT: .LBB3_12: ; %Flow19
+; GFX90A-NEXT: s_xor_b64 s[20:21], s[26:27], -1
+; GFX90A-NEXT: .LBB3_11: ; %Flow19
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_mov_b64 s[16:17], -1
-; GFX90A-NEXT: s_and_b64 vcc, exec, s[18:19]
+; GFX90A-NEXT: s_mov_b64 s[18:19], -1
+; GFX90A-NEXT: s_and_b64 vcc, exec, s[20:21]
; GFX90A-NEXT: s_cbranch_vccz .LBB3_1
-; GFX90A-NEXT: ; %bb.13: ; %bb12
+; GFX90A-NEXT: ; %bb.12: ; %bb12
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_add_u32 s2, s2, s8
-; GFX90A-NEXT: s_addc_u32 s3, s3, 0
-; GFX90A-NEXT: s_add_u32 s12, s12, s14
-; GFX90A-NEXT: s_addc_u32 s13, s13, s15
-; GFX90A-NEXT: s_mov_b64 s[16:17], 0
+; GFX90A-NEXT: s_add_u32 s4, s4, s8
+; GFX90A-NEXT: s_addc_u32 s5, s5, 0
+; GFX90A-NEXT: s_add_u32 s10, s10, s12
+; GFX90A-NEXT: s_addc_u32 s11, s11, s13
+; GFX90A-NEXT: s_mov_b64 s[18:19], 0
; GFX90A-NEXT: s_branch .LBB3_1
-; GFX90A-NEXT: .LBB3_14: ; %DummyReturnBlock
+; GFX90A-NEXT: .LBB3_13: ; %DummyReturnBlock
; GFX90A-NEXT: s_endpgm
bb:
%i = load volatile i16, ptr addrspace(4) poison, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/global_smrd_cfg.ll b/llvm/test/CodeGen/AMDGPU/global_smrd_cfg.ll
index f2255290e65a1..70af502b6d336 100644
--- a/llvm/test/CodeGen/AMDGPU/global_smrd_cfg.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_smrd_cfg.ll
@@ -9,9 +9,8 @@
; #####################################################################
; Load from %arg1 has no-alias store in Loop - arg1[i+1] never alias arg1[i]
-; However, our analysis cannot detect this.
-; CHECK: flat_load_dword
+; CHECK: s_load_dword
; #####################################################################
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-addsubu64.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-addsubu64.ll
index b557bc6d29a7e..82303c687c909 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-addsubu64.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-addsubu64.ll
@@ -32,7 +32,7 @@ define amdgpu_kernel void @add_reg_reg(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s64) from %ir.ptr.load, addrspace 1)
- ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR1:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile load (s64) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR1:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s64) from %ir.ptr.load, addrspace 1)
; CHECK-NEXT: [[V_ADD_U:%[0-9]+]]:vreg_64 = V_ADD_U64_PSEUDO [[GLOBAL_LOAD_DWORDX2_SADDR]], [[GLOBAL_LOAD_DWORDX2_SADDR1]], implicit-def $vcc_lo, implicit $exec
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY [[V_ADD_U]]
; CHECK-NEXT: GLOBAL_STORE_DWORDX2_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s64) into %ir.ptr.load, addrspace 1)
@@ -97,7 +97,7 @@ define amdgpu_kernel void @sub_reg_reg(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s64) from %ir.ptr.load, addrspace 1)
- ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR1:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile load (s64) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_SADDR1:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s64) from %ir.ptr.load, addrspace 1)
; CHECK-NEXT: [[V_SUB_U:%[0-9]+]]:vreg_64 = V_SUB_U64_PSEUDO [[GLOBAL_LOAD_DWORDX2_SADDR]], [[GLOBAL_LOAD_DWORDX2_SADDR1]], implicit-def $vcc_lo, implicit $exec
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY [[V_SUB_U]]
; CHECK-NEXT: GLOBAL_STORE_DWORDX2_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s64) into %ir.ptr.load, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
index 36ae9b583b003..46760dc7ee480 100644
--- a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgpu9.00 -amdgpu-aa -amdgpu-aa-wrapper -amdgpu-annotate-uniform -S < %s | FileCheck %s
; RUN: llc -mtriple=amdgpu9.00 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GCN %s
@@ -133,7 +133,7 @@ define amdgpu_kernel void @memory_phi_clobber1(ptr addrspace(1) %arg, i1 %cond)
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
; CHECK-NEXT: [[I1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 1, !amdgpu.uniform [[META0]]
-; CHECK-NEXT: [[I2:%.*]] = load i32, ptr addrspace(1) [[I1]], align 4
+; CHECK-NEXT: [[I2:%.*]] = load i32, ptr addrspace(1) [[I1]], align 4, !amdgpu.noclobber [[META0]]
; CHECK-NEXT: [[I3:%.*]] = add i32 [[I2]], [[I]]
; CHECK-NEXT: [[I4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 2
; CHECK-NEXT: store i32 [[I3]], ptr addrspace(1) [[I4]], align 4
@@ -162,10 +162,11 @@ define amdgpu_kernel void @memory_phi_clobber1(ptr addrspace(1) %arg, i1 %cond)
; GCN-NEXT: v_mov_b32_e32 v1, 1
; GCN-NEXT: global_store_dword v0, v1, s[0:1] offset:12
; GCN-NEXT: .LBB2_4: ; %if.end
+; GCN-NEXT: s_load_dword s2, s[0:1], 0x4
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: global_load_dword v1, v0, s[0:1] offset:4
-; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v1, s4, v1
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_add_i32 s2, s2, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: global_store_dword v0, v1, s[0:1] offset:8
; GCN-NEXT: s_endpgm
bb:
@@ -204,7 +205,7 @@ define amdgpu_kernel void @memory_phi_clobber2(ptr addrspace(1) %arg, i1 %cond)
; CHECK-NEXT: br label [[IF_END]]
; CHECK: if.end:
; CHECK-NEXT: [[I1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 1, !amdgpu.uniform [[META0]]
-; CHECK-NEXT: [[I2:%.*]] = load i32, ptr addrspace(1) [[I1]], align 4
+; CHECK-NEXT: [[I2:%.*]] = load i32, ptr addrspace(1) [[I1]], align 4, !amdgpu.noclobber [[META0]]
; CHECK-NEXT: [[I3:%.*]] = add i32 [[I2]], [[I]]
; CHECK-NEXT: [[I4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 2
; CHECK-NEXT: store i32 [[I3]], ptr addrspace(1) [[I4]], align 4
@@ -233,10 +234,11 @@ define amdgpu_kernel void @memory_phi_clobber2(ptr addrspace(1) %arg, i1 %cond)
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN-NEXT: s_barrier
; GCN-NEXT: .LBB3_4: ; %if.end
+; GCN-NEXT: s_load_dword s2, s[0:1], 0x4
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: global_load_dword v1, v0, s[0:1] offset:4
-; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v1, s4, v1
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_add_i32 s2, s2, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: global_store_dword v0, v1, s[0:1] offset:8
; GCN-NEXT: s_endpgm
bb:
@@ -828,5 +830,46 @@ entry:
ret void
}
+define amdgpu_kernel void @fence_in_loop(ptr addrspace(3) %p3, ptr addrspace(1) %p1) {
+; CHECK-LABEL: define amdgpu_kernel void @fence_in_loop(
+; CHECK-SAME: ptr addrspace(3) [[P3:%.*]], ptr addrspace(1) [[P1:%.*]]) {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: store i32 0, ptr addrspace(3) [[P3]], align 4
+; CHECK-NEXT: br label %[[BB1:.*]]
+; CHECK: [[BB1]]:
+; CHECK-NEXT: [[I:%.*]] = load i32, ptr addrspace(1) [[P1]], align 4, !amdgpu.noclobber [[META0]]
+; CHECK-NEXT: call void (...) @llvm.fake.use(i32 [[I]])
+; CHECK-NEXT: fence release
+; CHECK-NEXT: br label %[[BB1]]
+;
+; GCN-LABEL: fence_in_loop:
+; GCN: ; %bb.0: ; %bb
+; GCN-NEXT: s_load_dword s2, s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: s_and_b64 vcc, exec, -1
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: ds_write_b32 v1, v0
+; GCN-NEXT: .LBB18_1: ; %bb1
+; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
+; GCN-NEXT: s_load_dword s2, s[0:1], 0x0
+; GCN-NEXT: ; fake_use: $sgpr2
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b64 vcc, vcc
+; GCN-NEXT: s_cbranch_vccnz .LBB18_1
+; GCN-NEXT: ; %bb.2: ; %DummyReturnBlock
+; GCN-NEXT: s_endpgm
+bb:
+ store i32 0, ptr addrspace(3) %p3
+ br label %bb1
+
+bb1:
+ %i = load i32, ptr addrspace(1) %p1, align 4
+ call void (...) @llvm.fake.use(i32 %i)
+ fence release
+ br label %bb1
+}
+
declare void @llvm.amdgcn.s.barrier()
declare void @llvm.amdgcn.wave.barrier()
More information about the llvm-commits
mailing list