[llvm] [AMDGPU] Disable negative imm offset for async load/store instructions (PR #185078)
Shilei Tian via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 6 12:57:59 PST 2026
https://github.com/shiltian updated https://github.com/llvm/llvm-project/pull/185078
>From 2927c975a816a896b1302a9f4daa1548e782f40f Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Fri, 6 Mar 2026 13:40:36 -0500
Subject: [PATCH 1/2] [AMDGPU] Disable negative imm offset for async load/store
instructions
---
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 4 ++-
.../promote-constOffset-to-imm-gfx12.ll | 13 ++++---
.../promote-constOffset-to-imm-gfx12.mir | 36 ++++++++++---------
3 files changed, 29 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 0141c365789ca..6768af8abff99 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -2507,6 +2507,7 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
AM.HasBaseReg = true;
AM.BaseOffs = Dist;
if (TLI->isLegalFlatAddressingMode(AM, AS) &&
+ (!TII->usesASYNC_CNT(MI) || Dist >= 0) &&
(uint32_t)std::abs(Dist) > MaxDist) {
MaxDist = std::abs(Dist);
@@ -2534,7 +2535,8 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
AM.HasBaseReg = true;
AM.BaseOffs = OtherOffset - AnchorAddr.Offset;
- if (TLI->isLegalFlatAddressingMode(AM, AS)) {
+ if (TLI->isLegalFlatAddressingMode(AM, AS) &&
+ (!TII->usesASYNC_CNT(*OtherMI) || AM.BaseOffs >= 0)) {
LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset; dbgs() << ")";
OtherMI->dump());
int32_t OtherOffsetDiff = OtherOffset - AnchorAddr.Offset;
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
index b5b0164969212..9392bbe7c7a90 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @promote_async_load_offset_negative(ptr addrspace(1) %
; GFX1250-LABEL: promote_async_load_offset_negative:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
@@ -57,17 +57,16 @@ define amdgpu_kernel void @promote_async_load_offset_positive(ptr addrspace(1) %
; GFX1250-LABEL: promote_async_load_offset_positive:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v0, s[0:1]
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
-; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0x100, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x100, v[2:3]
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], 0x200, v[2:3]
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v[2:3], off offset:-256
+; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[4:5], off
; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
; GFX1250-NEXT: s_endpgm
entry:
@@ -103,7 +102,7 @@ define amdgpu_kernel void @promote_async_store_offset_negative(ptr addrspace(1)
; GFX1250-LABEL: promote_async_store_offset_negative:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, 0x100, v0
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
index dab161ba080e5..aea6fc443664b 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
@@ -14,11 +14,12 @@ body: |
; GFX1250: liveins: $ttmp7, $vgpr0, $sgpr0_sgpr1
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, 512, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr3, dead $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 256, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 256, $vgpr1, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 512, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
@@ -47,13 +48,15 @@ body: |
; GFX1250: liveins: $ttmp7, $vgpr0, $sgpr0_sgpr1
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, 516, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr3, dead $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 256, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 260, $vgpr1, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -260, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 4, $vgpr1, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -4, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 512, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 516, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
@@ -90,9 +93,9 @@ body: |
; GFX1250-NEXT: renamable $vgpr0 = V_AND_B32_e32 1023, killed $vgpr0, implicit $exec
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR $vgpr1, $sgpr0_sgpr1, $vgpr0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
; GFX1250-NEXT: renamable $vgpr2_vgpr3 = V_ADD_U64_e32 killed $sgpr0_sgpr1, $vgpr0_vgpr1, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e64 256, 0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr4_vgpr5 = V_ADD_U64_e32 256, $vgpr2_vgpr3, implicit $exec
; GFX1250-NEXT: renamable $vgpr2_vgpr3 = V_ADD_U64_e32 512, killed $vgpr2_vgpr3, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, -256, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr4_vgpr5, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
; GFX1250-NEXT: S_ENDPGM 0
%3:sgpr_64 = COPY $sgpr4_sgpr5
@@ -126,11 +129,12 @@ body: |
; GFX1250: liveins: $ttmp7, $vgpr0, $sgpr0_sgpr1
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 $vgpr0, 512, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr3, dead $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 256, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 256, $vgpr1, implicit $exec
- ; GFX1250-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128 $vgpr2_vgpr3, killed $vgpr0, -256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128 killed $vgpr2_vgpr3, $vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 512, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
; GFX1250-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128 killed $vgpr2_vgpr3, killed $vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
>From a870547b23777d5ca9b58096d958a1f594f00f09 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Fri, 6 Mar 2026 15:57:20 -0500
Subject: [PATCH 2/2] use smallest offset as anchor when negative offset is not
allowed
---
.../Target/AMDGPU/SILoadStoreOptimizer.cpp | 80 +++++-
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll | 74 +++--
.../CodeGen/AMDGPU/memintrinsic-unroll.ll | 270 +++++++-----------
.../promote-constOffset-to-imm-gfx12.ll | 9 +-
.../promote-constOffset-to-imm-gfx12.mir | 83 ++++--
5 files changed, 267 insertions(+), 249 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 6768af8abff99..6bee56898e4bf 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -2410,6 +2410,12 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
unsigned AS = SIInstrInfo::isFLATGlobal(MI) ? AMDGPUAS::GLOBAL_ADDRESS
: AMDGPUAS::FLAT_ADDRESS;
+ uint64_t FlatVariant = AS == AMDGPUAS::GLOBAL_ADDRESS
+ ? SIInstrFlags::FlatGlobal
+ : SIInstrFlags::FLAT;
+ bool AllowNegativeOffset =
+ TII->allowNegativeFlatOffset(FlatVariant) && !TII->usesASYNC_CNT(MI);
+
if (AnchorList.count(&MI))
return false;
@@ -2469,6 +2475,7 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
MemAddress AnchorAddr;
uint32_t MaxDist = std::numeric_limits<uint32_t>::min();
SmallVector<std::pair<MachineInstr *, int64_t>, 4> InstsWCommonBase;
+ bool MIIsAnchor = false;
MachineBasicBlock *MBB = MI.getParent();
MachineBasicBlock::iterator E = MBB->end();
@@ -2502,20 +2509,40 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
InstsWCommonBase.emplace_back(&MINext, MAddrNext.Offset);
- int64_t Dist = MAddr.Offset - MAddrNext.Offset;
- TargetLoweringBase::AddrMode AM;
- AM.HasBaseReg = true;
- AM.BaseOffs = Dist;
- if (TLI->isLegalFlatAddressingMode(AM, AS) &&
- (!TII->usesASYNC_CNT(MI) || Dist >= 0) &&
- (uint32_t)std::abs(Dist) > MaxDist) {
- MaxDist = std::abs(Dist);
-
- AnchorAddr = MAddrNext;
- AnchorInst = &MINext;
+ if (AllowNegativeOffset) {
+ int64_t Dist = MAddr.Offset - MAddrNext.Offset;
+ TargetLoweringBase::AddrMode AM;
+ AM.HasBaseReg = true;
+ AM.BaseOffs = Dist;
+ if (TLI->isLegalFlatAddressingMode(AM, AS) &&
+ (uint32_t)std::abs(Dist) > MaxDist) {
+ MaxDist = std::abs(Dist);
+
+ AnchorAddr = MAddrNext;
+ AnchorInst = &MINext;
+ }
}
}
+ // When negative offsets are not allowed, pick the candidate with the smallest
+ // offset as anchor so all promoted offsets are non-negative. If MI itself has
+ // the smallest offset, MI becomes the reference point (MIIsAnchor).
+ if (!AllowNegativeOffset && !InstsWCommonBase.empty()) {
+ for (auto &[Inst, Offset] : InstsWCommonBase) {
+ int64_t Dist = MAddr.Offset - Offset;
+ TargetLoweringBase::AddrMode AM;
+ AM.HasBaseReg = true;
+ AM.BaseOffs = Dist;
+ if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AS) &&
+ (!AnchorInst || Offset < AnchorAddr.Offset)) {
+ AnchorAddr = Visited[Inst];
+ AnchorInst = Inst;
+ }
+ }
+ if (!AnchorInst)
+ MIIsAnchor = true;
+ }
+
if (AnchorInst) {
LLVM_DEBUG(dbgs() << " Anchor-Inst(with max-distance from Offset): ";
AnchorInst->dump());
@@ -2536,7 +2563,7 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
AM.BaseOffs = OtherOffset - AnchorAddr.Offset;
if (TLI->isLegalFlatAddressingMode(AM, AS) &&
- (!TII->usesASYNC_CNT(*OtherMI) || AM.BaseOffs >= 0)) {
+ (AllowNegativeOffset || AM.BaseOffs >= 0)) {
LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset; dbgs() << ")";
OtherMI->dump());
int32_t OtherOffsetDiff = OtherOffset - AnchorAddr.Offset;
@@ -2549,6 +2576,35 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
return true;
}
+ if (MIIsAnchor) {
+ LLVM_DEBUG(dbgs() << " MI is anchor (smallest offset); promoting "
+ "candidates relative to MI's base.\n");
+
+ Register Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->getReg();
+ bool AnyPromoted = false;
+
+ for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
+ int64_t Dist = OtherOffset - MAddr.Offset;
+ TargetLoweringBase::AddrMode AM;
+ AM.HasBaseReg = true;
+ AM.BaseOffs = Dist;
+ if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AS)) {
+ LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset << ")";
+ OtherMI->dump());
+ updateBaseAndOffset(*OtherMI, Base, Dist);
+ updateAsyncLDSAddress(*OtherMI, Dist);
+ LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
+ AnyPromoted = true;
+ }
+ }
+
+ if (AnyPromoted) {
+ TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->setIsKill(false);
+ AnchorList.insert(&MI);
+ return true;
+ }
+ }
+
return false;
}
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index af753812b4c0c..57b653c29ae68 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -495,34 +495,31 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e64 v6, s[4:5], 28, v0
-; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, v1, s[4:5]
-; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e64 v8, s[4:5], 44, v0
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
-; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e64 v9, s[4:5], 0, v1, s[4:5]
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[10:13], v[8:9] offset:16
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[14:17], v[8:9] offset:32
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[18:21], v[8:9] offset:48
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[22:25], v[8:9] offset:64
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[26:29], v[8:9] offset:80
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[30:33], v[6:7]
-; GFX90A-SDAG-MUBUF-NEXT: s_nop 0
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[6:9], v[8:9]
+; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e64 v7, s[4:5], 0, v1, s[4:5]
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[34:37], v[0:1] offset:12
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
; GFX90A-SDAG-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[22:25] offset:48
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[26:29] offset:64
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[14:17] offset:64
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[18:21] offset:32
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[6:9] offset:32
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[10:13]
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[30:33] offset:16
-; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[34:37]
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
+; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
; GFX90A-SDAG-MUBUF-NEXT: flat_store_dword v[4:5], v50
; GFX90A-SDAG-MUBUF-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
; GFX90A-SDAG-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -536,34 +533,31 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e64 v6, s[0:1], 28, v0
-; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e64 v7, s[0:1], 0, v1, s[0:1]
-; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e64 v8, s[0:1], 44, v0
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x8c, v0
-; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e64 v9, s[0:1], 0, v1, s[0:1]
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[10:13], v[8:9] offset:16
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[14:17], v[8:9] offset:32
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[18:21], v[8:9] offset:48
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[22:25], v[8:9] offset:64
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[26:29], v[8:9] offset:80
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[30:33], v[6:7]
-; GFX90A-SDAG-FLATSCR-NEXT: s_nop 0
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[6:9], v[8:9]
+; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e64 v7, s[0:1], 0, v1, s[0:1]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[8:11], v[6:7] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[12:15], v[6:7] offset:48
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[16:19], v[6:7] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[20:23], v[6:7] offset:80
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[24:27], v[6:7] offset:96
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[28:31], v[6:7]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[34:37], v[0:1] offset:12
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
; GFX90A-SDAG-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[22:25] offset:48
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[26:29] offset:64
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[14:17] offset:64
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[18:21] offset:32
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[6:9] offset:32
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[10:13]
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[30:33] offset:16
-; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[34:37]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[20:23] offset:48
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[24:27] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[32:35] offset:32
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[0:1], v[8:11]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[28:31] offset:16
+; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx4 v[2:3], v[36:39]
; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dword v[4:5], v50
; GFX90A-SDAG-FLATSCR-NEXT: flat_store_dwordx2 v[2:3], v[48:49] offset:128
; GFX90A-SDAG-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index c60642e2cc4d8..a2443bf25ef31 100644
--- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -16,31 +16,26 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: .LBB0_1: ; %static-memcpy-expansion-main-body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v24, vcc_lo, v2, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v25, null, s5, v3, vcc_lo
-; CHECK-NEXT: s_clause 0x4
-; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[24:25] offset:64
-; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[24:25] offset:32
-; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[24:25]
-; CHECK-NEXT: flat_load_dwordx4 v[16:19], v[24:25] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[24:25] offset:128
-; CHECK-NEXT: v_add_co_u32 v80, vcc_lo, v24, 0x60
-; CHECK-NEXT: v_add_co_ci_u32_e64 v81, null, 0, v25, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v96, vcc_lo, v24, 48
-; CHECK-NEXT: v_add_co_ci_u32_e64 v97, null, 0, v25, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v84, vcc_lo, 0x50, v24
-; CHECK-NEXT: v_add_co_ci_u32_e64 v85, null, 0, v25, vcc_lo
-; CHECK-NEXT: s_clause 0xa
-; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[80:81] offset:128
-; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[80:81] offset:144
-; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[80:81] offset:96
-; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[80:81] offset:112
-; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[80:81] offset:64
-; CHECK-NEXT: flat_load_dwordx4 v[52:55], v[80:81] offset:80
-; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[80:81] offset:48
-; CHECK-NEXT: flat_load_dwordx4 v[68:71], v[80:81]
-; CHECK-NEXT: flat_load_dwordx4 v[80:83], v[80:81] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[84:87], v[84:85]
+; CHECK-NEXT: v_add_co_u32 v20, vcc_lo, v2, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v21, null, s5, v3, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v96, vcc_lo, v20, 48
+; CHECK-NEXT: v_add_co_ci_u32_e64 v97, null, 0, v21, vcc_lo
+; CHECK-NEXT: s_clause 0xf
+; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[20:21] offset:64
+; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[20:21] offset:32
+; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[20:21]
+; CHECK-NEXT: flat_load_dwordx4 v[16:19], v[20:21] offset:16
+; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[20:21] offset:128
+; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[96:97] offset:176
+; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[96:97] offset:192
+; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[96:97] offset:144
+; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[96:97] offset:160
+; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[96:97] offset:112
+; CHECK-NEXT: flat_load_dwordx4 v[52:55], v[96:97] offset:128
+; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[96:97] offset:96
+; CHECK-NEXT: flat_load_dwordx4 v[68:71], v[96:97] offset:48
+; CHECK-NEXT: flat_load_dwordx4 v[80:83], v[96:97] offset:64
+; CHECK-NEXT: flat_load_dwordx4 v[84:87], v[96:97] offset:32
; CHECK-NEXT: flat_load_dwordx4 v[96:99], v[96:97]
; CHECK-NEXT: v_add_co_u32 v100, vcc_lo, v0, s4
; CHECK-NEXT: s_add_u32 s4, s4, 0x100
@@ -112,45 +107,25 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: ; =>This Inner Loop Header: Depth=1
; ALIGNED-NEXT: v_add_co_u32 v4, vcc_lo, v2, s4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v5, null, s5, v3, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v6, vcc_lo, v4, 48
+; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, v4, 48
; ALIGNED-NEXT: flat_load_dwordx4 v[20:23], v[4:5] offset:128
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v8, vcc_lo, 0x50, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v12, vcc_lo, 0x60, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v13, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v16, vcc_lo, 0x70, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, 0x90, v4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v52, vcc_lo, 0xa0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v53, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v54, vcc_lo, 0xb0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v55, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v64, vcc_lo, 0xc0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v65, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v66, vcc_lo, 0xd0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v67, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v68, vcc_lo, 0xe0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v69, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v70, vcc_lo, 0xf0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v71, null, 0, v5, vcc_lo
; ALIGNED-NEXT: s_clause 0xe
; ALIGNED-NEXT: flat_load_dwordx4 v[26:29], v[4:5]
; ALIGNED-NEXT: flat_load_dwordx4 v[35:38], v[4:5] offset:16
; ALIGNED-NEXT: flat_load_dwordx4 v[30:33], v[4:5] offset:32
; ALIGNED-NEXT: flat_load_dwordx4 v[48:51], v[4:5] offset:64
-; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[6:7]
-; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
-; ALIGNED-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
-; ALIGNED-NEXT: flat_load_dwordx4 v[112:115], v[24:25]
-; ALIGNED-NEXT: flat_load_dwordx4 v[116:119], v[52:53]
-; ALIGNED-NEXT: flat_load_dwordx4 v[40:43], v[54:55]
-; ALIGNED-NEXT: flat_load_dwordx4 v[44:47], v[64:65]
-; ALIGNED-NEXT: flat_load_dwordx4 v[55:58], v[66:67]
-; ALIGNED-NEXT: flat_load_dwordx4 v[59:62], v[68:69]
-; ALIGNED-NEXT: flat_load_dwordx4 v[68:71], v[70:71]
+; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[24:25]
+; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[24:25] offset:32
+; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[24:25] offset:48
+; ALIGNED-NEXT: flat_load_dwordx4 v[16:19], v[24:25] offset:64
+; ALIGNED-NEXT: flat_load_dwordx4 v[112:115], v[24:25] offset:96
+; ALIGNED-NEXT: flat_load_dwordx4 v[116:119], v[24:25] offset:112
+; ALIGNED-NEXT: flat_load_dwordx4 v[40:43], v[24:25] offset:128
+; ALIGNED-NEXT: flat_load_dwordx4 v[44:47], v[24:25] offset:144
+; ALIGNED-NEXT: flat_load_dwordx4 v[55:58], v[24:25] offset:160
+; ALIGNED-NEXT: flat_load_dwordx4 v[59:62], v[24:25] offset:176
+; ALIGNED-NEXT: flat_load_dwordx4 v[68:71], v[24:25] offset:192
; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, v0, s4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, s5, v1, vcc_lo
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
@@ -5410,29 +5385,24 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_add_co_u32 v52, vcc_lo, v2, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v53, null, s5, v3, vcc_lo
-; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[52:53] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[52:53]
; CHECK-NEXT: v_add_co_u32 v96, vcc_lo, v52, 48
; CHECK-NEXT: v_add_co_ci_u32_e64 v97, null, 0, v53, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v80, vcc_lo, v52, 0x60
-; CHECK-NEXT: v_add_co_ci_u32_e64 v81, null, 0, v53, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v84, vcc_lo, 0x50, v52
-; CHECK-NEXT: v_add_co_ci_u32_e64 v85, null, 0, v53, vcc_lo
-; CHECK-NEXT: s_clause 0xd
-; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[52:53] offset:32
-; CHECK-NEXT: flat_load_dwordx4 v[16:19], v[52:53] offset:64
-; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[80:81] offset:128
-; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[80:81] offset:144
-; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[80:81] offset:96
-; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[80:81] offset:112
-; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[80:81] offset:64
-; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[80:81] offset:80
+; CHECK-NEXT: s_clause 0xf
+; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[52:53] offset:64
+; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[52:53] offset:32
+; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[52:53]
+; CHECK-NEXT: flat_load_dwordx4 v[16:19], v[52:53] offset:16
+; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[96:97] offset:176
+; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[96:97] offset:192
+; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[96:97] offset:144
+; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[96:97] offset:160
+; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[96:97] offset:112
+; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[96:97] offset:128
; CHECK-NEXT: flat_load_dwordx4 v[52:55], v[52:53] offset:128
-; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[80:81] offset:48
-; CHECK-NEXT: flat_load_dwordx4 v[68:71], v[80:81]
-; CHECK-NEXT: flat_load_dwordx4 v[80:83], v[80:81] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[84:87], v[84:85]
+; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[96:97] offset:96
+; CHECK-NEXT: flat_load_dwordx4 v[68:71], v[96:97] offset:48
+; CHECK-NEXT: flat_load_dwordx4 v[80:83], v[96:97] offset:64
+; CHECK-NEXT: flat_load_dwordx4 v[84:87], v[96:97] offset:32
; CHECK-NEXT: flat_load_dwordx4 v[96:99], v[96:97]
; CHECK-NEXT: v_add_co_u32 v100, vcc_lo, v0, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v101, null, s5, v1, vcc_lo
@@ -5441,11 +5411,14 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; CHECK-NEXT: v_add_co_u32 v102, vcc_lo, v100, 48
; CHECK-NEXT: s_cmp_lg_u64 s[4:5], 0x800
; CHECK-NEXT: v_add_co_ci_u32_e64 v103, null, 0, v101, vcc_lo
-; CHECK-NEXT: s_waitcnt vmcnt(12) lgkmcnt(12)
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[16:19] offset:64
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[12:15] offset:32
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[8:11]
-; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[4:7] offset:16
+; CHECK-NEXT: s_waitcnt vmcnt(15) lgkmcnt(15)
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[4:7] offset:64
+; CHECK-NEXT: s_waitcnt vmcnt(14) lgkmcnt(15)
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[8:11] offset:32
+; CHECK-NEXT: s_waitcnt vmcnt(13) lgkmcnt(15)
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[12:15]
+; CHECK-NEXT: s_waitcnt vmcnt(12) lgkmcnt(15)
+; CHECK-NEXT: flat_store_dwordx4 v[100:101], v[16:19] offset:16
; CHECK-NEXT: s_waitcnt vmcnt(11) lgkmcnt(15)
; CHECK-NEXT: flat_store_dwordx4 v[102:103], v[20:23] offset:176
; CHECK-NEXT: s_waitcnt vmcnt(10) lgkmcnt(15)
@@ -5480,31 +5453,26 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; CHECK-NEXT: s_mov_b32 s7, -1
; CHECK-NEXT: .LBB5_5: ; %memmove_bwd_loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v24, vcc_lo, v2, s4
-; CHECK-NEXT: v_add_co_ci_u32_e64 v25, null, s5, v3, vcc_lo
-; CHECK-NEXT: s_clause 0x4
-; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[24:25] offset:64
-; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[24:25] offset:32
-; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[24:25]
-; CHECK-NEXT: flat_load_dwordx4 v[16:19], v[24:25] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[24:25] offset:128
-; CHECK-NEXT: v_add_co_u32 v80, vcc_lo, v24, 0x60
-; CHECK-NEXT: v_add_co_ci_u32_e64 v81, null, 0, v25, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v96, vcc_lo, v24, 48
-; CHECK-NEXT: v_add_co_ci_u32_e64 v97, null, 0, v25, vcc_lo
-; CHECK-NEXT: v_add_co_u32 v84, vcc_lo, 0x50, v24
-; CHECK-NEXT: v_add_co_ci_u32_e64 v85, null, 0, v25, vcc_lo
-; CHECK-NEXT: s_clause 0xa
-; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[80:81] offset:128
-; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[80:81] offset:144
-; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[80:81] offset:96
-; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[80:81] offset:112
-; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[80:81] offset:64
-; CHECK-NEXT: flat_load_dwordx4 v[52:55], v[80:81] offset:80
-; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[80:81] offset:48
-; CHECK-NEXT: flat_load_dwordx4 v[68:71], v[80:81]
-; CHECK-NEXT: flat_load_dwordx4 v[80:83], v[80:81] offset:16
-; CHECK-NEXT: flat_load_dwordx4 v[84:87], v[84:85]
+; CHECK-NEXT: v_add_co_u32 v20, vcc_lo, v2, s4
+; CHECK-NEXT: v_add_co_ci_u32_e64 v21, null, s5, v3, vcc_lo
+; CHECK-NEXT: v_add_co_u32 v96, vcc_lo, v20, 48
+; CHECK-NEXT: v_add_co_ci_u32_e64 v97, null, 0, v21, vcc_lo
+; CHECK-NEXT: s_clause 0xf
+; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[20:21] offset:64
+; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[20:21] offset:32
+; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[20:21]
+; CHECK-NEXT: flat_load_dwordx4 v[16:19], v[20:21] offset:16
+; CHECK-NEXT: flat_load_dwordx4 v[20:23], v[20:21] offset:128
+; CHECK-NEXT: flat_load_dwordx4 v[24:27], v[96:97] offset:176
+; CHECK-NEXT: flat_load_dwordx4 v[28:31], v[96:97] offset:192
+; CHECK-NEXT: flat_load_dwordx4 v[32:35], v[96:97] offset:144
+; CHECK-NEXT: flat_load_dwordx4 v[36:39], v[96:97] offset:160
+; CHECK-NEXT: flat_load_dwordx4 v[48:51], v[96:97] offset:112
+; CHECK-NEXT: flat_load_dwordx4 v[52:55], v[96:97] offset:128
+; CHECK-NEXT: flat_load_dwordx4 v[64:67], v[96:97] offset:96
+; CHECK-NEXT: flat_load_dwordx4 v[68:71], v[96:97] offset:48
+; CHECK-NEXT: flat_load_dwordx4 v[80:83], v[96:97] offset:64
+; CHECK-NEXT: flat_load_dwordx4 v[84:87], v[96:97] offset:32
; CHECK-NEXT: flat_load_dwordx4 v[96:99], v[96:97]
; CHECK-NEXT: v_add_co_u32 v100, vcc_lo, v0, s4
; CHECK-NEXT: v_add_co_ci_u32_e64 v101, null, s5, v1, vcc_lo
@@ -5581,45 +5549,25 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: ; =>This Inner Loop Header: Depth=1
; ALIGNED-NEXT: v_add_co_u32 v4, vcc_lo, v2, s4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v5, null, s5, v3, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v6, vcc_lo, v4, 48
+; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, v4, 48
; ALIGNED-NEXT: flat_load_dwordx4 v[20:23], v[4:5] offset:128
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v8, vcc_lo, 0x50, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v12, vcc_lo, 0x60, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v13, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v16, vcc_lo, 0x70, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, 0x90, v4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v38, vcc_lo, 0xa0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v39, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v52, vcc_lo, 0xb0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v53, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v54, vcc_lo, 0xc0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v55, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v64, vcc_lo, 0xd0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v65, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v70, vcc_lo, 0xe0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v71, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v80, vcc_lo, 0xf0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v81, null, 0, v5, vcc_lo
; ALIGNED-NEXT: s_clause 0xe
; ALIGNED-NEXT: flat_load_dwordx4 v[34:37], v[4:5]
; ALIGNED-NEXT: flat_load_dwordx4 v[30:33], v[4:5] offset:16
; ALIGNED-NEXT: flat_load_dwordx4 v[26:29], v[4:5] offset:32
; ALIGNED-NEXT: flat_load_dwordx4 v[48:51], v[4:5] offset:64
-; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[6:7]
-; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
-; ALIGNED-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
-; ALIGNED-NEXT: flat_load_dwordx4 v[112:115], v[24:25]
-; ALIGNED-NEXT: flat_load_dwordx4 v[82:85], v[38:39]
-; ALIGNED-NEXT: flat_load_dwordx4 v[116:119], v[52:53]
-; ALIGNED-NEXT: flat_load_dwordx4 v[66:69], v[54:55]
-; ALIGNED-NEXT: flat_load_dwordx4 v[40:43], v[64:65]
-; ALIGNED-NEXT: flat_load_dwordx4 v[52:55], v[70:71]
-; ALIGNED-NEXT: flat_load_dwordx4 v[44:47], v[80:81]
+; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[24:25]
+; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[24:25] offset:32
+; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[24:25] offset:48
+; ALIGNED-NEXT: flat_load_dwordx4 v[16:19], v[24:25] offset:64
+; ALIGNED-NEXT: flat_load_dwordx4 v[112:115], v[24:25] offset:96
+; ALIGNED-NEXT: flat_load_dwordx4 v[82:85], v[24:25] offset:112
+; ALIGNED-NEXT: flat_load_dwordx4 v[116:119], v[24:25] offset:128
+; ALIGNED-NEXT: flat_load_dwordx4 v[66:69], v[24:25] offset:144
+; ALIGNED-NEXT: flat_load_dwordx4 v[40:43], v[24:25] offset:160
+; ALIGNED-NEXT: flat_load_dwordx4 v[52:55], v[24:25] offset:176
+; ALIGNED-NEXT: flat_load_dwordx4 v[44:47], v[24:25] offset:192
; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, v0, s4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, s5, v1, vcc_lo
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
@@ -6236,45 +6184,25 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: ; =>This Inner Loop Header: Depth=1
; ALIGNED-NEXT: v_add_co_u32 v4, vcc_lo, v2, s4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v5, null, s5, v3, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v6, vcc_lo, v4, 48
+; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, v4, 48
; ALIGNED-NEXT: flat_load_dwordx4 v[20:23], v[4:5] offset:128
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v8, vcc_lo, 0x50, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v12, vcc_lo, 0x60, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v13, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v16, vcc_lo, 0x70, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, 0x90, v4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v52, vcc_lo, 0xa0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v53, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v54, vcc_lo, 0xb0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v55, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v64, vcc_lo, 0xc0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v65, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v66, vcc_lo, 0xd0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v67, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v68, vcc_lo, 0xe0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v69, null, 0, v5, vcc_lo
-; ALIGNED-NEXT: v_add_co_u32 v70, vcc_lo, 0xf0, v4
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v71, null, 0, v5, vcc_lo
; ALIGNED-NEXT: s_clause 0xe
; ALIGNED-NEXT: flat_load_dwordx4 v[26:29], v[4:5]
; ALIGNED-NEXT: flat_load_dwordx4 v[35:38], v[4:5] offset:16
; ALIGNED-NEXT: flat_load_dwordx4 v[30:33], v[4:5] offset:32
; ALIGNED-NEXT: flat_load_dwordx4 v[48:51], v[4:5] offset:64
-; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[6:7]
-; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
-; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
-; ALIGNED-NEXT: flat_load_dwordx4 v[16:19], v[16:17]
-; ALIGNED-NEXT: flat_load_dwordx4 v[112:115], v[24:25]
-; ALIGNED-NEXT: flat_load_dwordx4 v[116:119], v[52:53]
-; ALIGNED-NEXT: flat_load_dwordx4 v[40:43], v[54:55]
-; ALIGNED-NEXT: flat_load_dwordx4 v[44:47], v[64:65]
-; ALIGNED-NEXT: flat_load_dwordx4 v[55:58], v[66:67]
-; ALIGNED-NEXT: flat_load_dwordx4 v[59:62], v[68:69]
-; ALIGNED-NEXT: flat_load_dwordx4 v[68:71], v[70:71]
+; ALIGNED-NEXT: flat_load_dwordx4 v[4:7], v[24:25]
+; ALIGNED-NEXT: flat_load_dwordx4 v[8:11], v[24:25] offset:32
+; ALIGNED-NEXT: flat_load_dwordx4 v[12:15], v[24:25] offset:48
+; ALIGNED-NEXT: flat_load_dwordx4 v[16:19], v[24:25] offset:64
+; ALIGNED-NEXT: flat_load_dwordx4 v[112:115], v[24:25] offset:96
+; ALIGNED-NEXT: flat_load_dwordx4 v[116:119], v[24:25] offset:112
+; ALIGNED-NEXT: flat_load_dwordx4 v[40:43], v[24:25] offset:128
+; ALIGNED-NEXT: flat_load_dwordx4 v[44:47], v[24:25] offset:144
+; ALIGNED-NEXT: flat_load_dwordx4 v[55:58], v[24:25] offset:160
+; ALIGNED-NEXT: flat_load_dwordx4 v[59:62], v[24:25] offset:176
+; ALIGNED-NEXT: flat_load_dwordx4 v[68:71], v[24:25] offset:192
; ALIGNED-NEXT: v_add_co_u32 v24, vcc_lo, v0, s4
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v25, null, s5, v1, vcc_lo
; ALIGNED-NEXT: s_add_u32 s4, s4, 0xffffff00
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
index 9392bbe7c7a90..e52a9739c45f4 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.ll
@@ -63,11 +63,12 @@ define amdgpu_kernel void @promote_async_load_offset_positive(ptr addrspace(1) %
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v0, s[0:1]
; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x100, v[2:3]
-; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], 0x200, v[2:3]
-; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[4:5], off
+; GFX1250-NEXT: v_add_nc_u32_e64 v0, 0xffffff00, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[2:3], 0x100, v[2:3]
+; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_load_async_to_lds_b128 v1, v[2:3], off
+; GFX1250-NEXT: global_load_async_to_lds_b128 v0, v[2:3], off offset:256
; GFX1250-NEXT: s_endpgm
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
index aea6fc443664b..9a1e6c006308f 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm-gfx12.mir
@@ -15,12 +15,11 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 256, $vgpr0, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 512, $vgpr0, 0, implicit $exec
; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 -256, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, killed $vgpr2_vgpr3, 256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
%2:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
@@ -49,15 +48,13 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 256, $vgpr0, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 512, $vgpr0, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 516, $vgpr0, 0, implicit $exec
; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 -256, $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, 256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 -260, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, killed $vgpr2_vgpr3, 260, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
%2:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
@@ -93,10 +90,10 @@ body: |
; GFX1250-NEXT: renamable $vgpr0 = V_AND_B32_e32 1023, killed $vgpr0, implicit $exec
; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR $vgpr1, $sgpr0_sgpr1, $vgpr0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
; GFX1250-NEXT: renamable $vgpr2_vgpr3 = V_ADD_U64_e32 killed $sgpr0_sgpr1, $vgpr0_vgpr1, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr4_vgpr5 = V_ADD_U64_e32 256, $vgpr2_vgpr3, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr2_vgpr3 = V_ADD_U64_e32 512, killed $vgpr2_vgpr3, implicit $exec
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, killed $vgpr4_vgpr5, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, killed $vgpr2_vgpr3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e64 -256, 0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2_vgpr3 = V_ADD_U64_e32 256, killed $vgpr2_vgpr3, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr1, $vgpr2_vgpr3, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, killed $vgpr2_vgpr3, 256, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
; GFX1250-NEXT: S_ENDPGM 0
%3:sgpr_64 = COPY $sgpr4_sgpr5
%0:vgpr_32 = COPY $vgpr0
@@ -113,6 +110,49 @@ body: |
S_ENDPGM 0
...
+# Three loads in ascending offset order. MI (base+256) has the smallest offset
+# among forward candidates, so MI becomes the anchor. Both base+512 and base+768
+# should be promoted to use MI's vaddr with positive offsets 256 and 512.
+---
+name: promote_async_load_mi_is_anchor
+machineFunctionInfo:
+ stackPtrOffsetReg: '$sgpr32'
+ frameOffsetReg: '$sgpr33'
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ liveins: $vgpr0, $sgpr0_sgpr1, $ttmp7
+ ; GFX1250-LABEL: name: promote_async_load_mi_is_anchor
+ ; GFX1250: liveins: $ttmp7, $vgpr0, $sgpr0_sgpr1
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 256, $vgpr0, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
+ ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 $vgpr1, $vgpr2_vgpr3, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 -256, $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, $vgpr2_vgpr3, 256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 -512, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed $vgpr0, killed $vgpr2_vgpr3, 512, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
+ %2:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
+ %3:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
+ %20:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
+ %4:vgpr_32, %5:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 256, 0, implicit $exec
+ %6:vgpr_32, %7:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %5, 0, implicit $exec
+ %8:vreg_64_align2 = REG_SEQUENCE %4, %subreg.sub0, %6, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %2, killed %8, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %9:vgpr_32, %10:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 512, 0, implicit $exec
+ %11:vgpr_32, %12:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %10, 0, implicit $exec
+ %13:vreg_64_align2 = REG_SEQUENCE %9, %subreg.sub0, %11, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %3, killed %13, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ %14:vgpr_32, %15:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 %0, 768, 0, implicit $exec
+ %16:vgpr_32, %17:sreg_32_xm0_xexec = V_ADDC_U32_e64 %0, 0, killed %15, 0, implicit $exec
+ %18:vreg_64_align2 = REG_SEQUENCE %14, %subreg.sub0, %16, %subreg.sub1
+ GLOBAL_LOAD_ASYNC_TO_LDS_B128 killed %20, killed %18, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+...
+
# Same as promote_async_load_offset above, but for async stores. The LDS address
# is in vdata instead of vdst, so this tests that updateAsyncLDSAddress corrects
# the right operand.
@@ -130,12 +170,11 @@ body: |
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: renamable $vgpr1 = V_LSHLREV_B32_e32 8, $vgpr0, implicit $exec
; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 256, $vgpr0, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, $vgpr0, killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
- ; GFX1250-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128 killed $vgpr2_vgpr3, $vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
- ; GFX1250-NEXT: renamable $vgpr2, renamable $vcc_lo = V_ADD_CO_U32_e64 512, $vgpr0, 0, implicit $exec
; GFX1250-NEXT: renamable $vgpr3, $sgpr_null = V_ADDC_U32_e64 0, killed $vgpr0, killed $vcc_lo, 0, implicit $exec
- ; GFX1250-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128 killed $vgpr2_vgpr3, killed $vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr1 = disjoint V_OR_B32_e32 0, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128 $vgpr2_vgpr3, $vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
+ ; GFX1250-NEXT: renamable $vgpr0 = V_ADD_U32_e32 -256, killed $vgpr1, implicit $exec
+ ; GFX1250-NEXT: GLOBAL_STORE_ASYNC_FROM_LDS_B128 killed $vgpr2_vgpr3, killed $vgpr0, 256, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt :: (load store (s128), align 1, addrspace 3)
%0:vgpr_32 = COPY $vgpr0
%1:vgpr_32 = V_LSHLREV_B32_e64 8, %0, implicit $exec
%2:vgpr_32 = disjoint V_OR_B32_e64 %1, 0, implicit $exec
More information about the llvm-commits
mailing list