[llvm] AMDGPU: Shrink extracted dword vector loads (PR #203484)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 02:27:41 PDT 2026
https://github.com/tianhbai created https://github.com/llvm/llvm-project/pull/203484
## Summary
Teach AMDGPU SelectionDAG to shrink simple dword vector loads that feed constant-index `extract_vector_elt` users.
When the used lanes form one contiguous run, the combine replaces the original load with one narrower `2`-`4` dword load at the adjusted byte offset and rewrites the extract users.
The transform stays conservative:
- only handles simple normal `i32` vector loads
- only shrinks one contiguous used-lane range
- only emits 2-4 dword loads
- checks `dwordx3`, local, and private load support
- skips loads that may select to SMRD, avoiding cases where this could block later `s_load_dwordx8` / `s_load_dwordx16` merging
This enables VMEM/flat/local/private cases such as `global_load_b128` to become `global_load_b96` when the unused dword is proven unnecessary.
## Tests
- Built `llc`
- `llvm-lit -sv llvm/test/CodeGen/AMDGPU`
## AI Tool Usage
AI tools were used to assist with code exploration, drafting, and review of this PR. I reviewed and validated the changes.
>From ee69fd751efc329b902736d24bccfa5d0838843c Mon Sep 17 00:00:00 2001
From: tianhbai <tianhbai at gmail.com>
Date: Fri, 12 Jun 2026 16:50:45 +0800
Subject: [PATCH] AMDGPU: Shrink extracted dword vector loads
Add a post-legalize SelectionDAG combine for simple i32 vector loads whose value users are constant-index extract_vector_elt nodes. When the used lanes form one contiguous range, replace the load with a narrower dword vector load and rewrite the extracts.
Keep the transform conservative by only emitting 2-4 dword loads, checking target support for dwordx3 loads, rejecting unsupported local/private access shapes, and avoiding loads that may select to SMRD so the combine does not block later wider scalar load merging.
Co-authored-by: Codex <noreply at openai.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 170 ++++++++-
llvm/test/CodeGen/AMDGPU/bf16.ll | 20 +-
.../CodeGen/AMDGPU/bug-v4f64-subvector.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll | 130 ++++---
llvm/test/CodeGen/AMDGPU/function-returns.ll | 4 +-
.../AMDGPU/gfx-callable-return-types.ll | 44 +--
llvm/test/CodeGen/AMDGPU/idot4u.ll | 154 ++++----
.../issue153808-extract-subvector-legalize.ll | 50 ++-
.../CodeGen/AMDGPU/llvm.amdgcn.readlane.ll | 8 +-
llvm/test/CodeGen/AMDGPU/permute_i8.ll | 16 +-
.../AMDGPU/shrink-extracted-dword-load.ll | 357 ++++++++++++++++++
llvm/test/CodeGen/AMDGPU/sra.ll | 2 +-
.../CodeGen/AMDGPU/vector_range_metadata.ll | 14 +-
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 84 ++---
14 files changed, 780 insertions(+), 277 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/shrink-extracted-dword-load.ll
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a6dc58e6da263..4562e329b6a6a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16505,6 +16505,171 @@ bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
EltSize, NumElem, Idx->isDivergent(), getSubtarget());
}
+// Return true if this load may select to an SMRD instruction.
+static bool maySelectSMRD(const LoadSDNode *Ld, const GCNSubtarget *Subtarget,
+ const SITargetLowering *TLI) {
+ const MachineMemOperand *MMO = Ld->getMemOperand();
+ if (!MMO->getSize().hasValue())
+ return false;
+
+ Align MinScalarAlign(
+ std::min(MMO->getSize().getValue().getKnownMinValue(), uint64_t(4)));
+ if (Ld->getAlign() < MinScalarAlign)
+ return false;
+
+ if (Ld->isDivergent() && !AMDGPU::isUniformMMO(MMO))
+ return false;
+
+ switch (Ld->getAddressSpace()) {
+ case AMDGPUAS::CONSTANT_ADDRESS:
+ case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
+ return true;
+ case AMDGPUAS::GLOBAL_ADDRESS:
+ return Ld->getMemOperand()->isInvariant() ||
+ (Subtarget->getScalarizeGlobalBehavior() &&
+ TLI->isMemOpHasNoClobberedMemOperand(Ld));
+ default:
+ return false;
+ }
+}
+
+static bool canShrinkDwordVectorLoad(const LoadSDNode *Ld,
+ const GCNSubtarget *Subtarget,
+ const SITargetLowering *TLI,
+ unsigned NewNumDwords,
+ uint64_t ByteOffset) {
+ // Keep this as one vector load: no scalar load and no wider-than-128-bit
+ // load.
+ if (NewNumDwords < 2 || NewNumDwords > 4)
+ return false;
+
+ // Avoid blocking later SMRD merging into wider x8/x16 loads.
+ if (maySelectSMRD(Ld, Subtarget, TLI))
+ return false;
+
+ if (NewNumDwords == 3 && !Subtarget->hasDwordx3LoadStores())
+ return false;
+
+ unsigned AS = Ld->getAddressSpace();
+ Align NewAlign = commonAlignment(Ld->getAlign(), ByteOffset);
+ switch (AS) {
+ case AMDGPUAS::CONSTANT_ADDRESS:
+ case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
+ case AMDGPUAS::GLOBAL_ADDRESS:
+ case AMDGPUAS::FLAT_ADDRESS:
+ return true;
+ case AMDGPUAS::LOCAL_ADDRESS: {
+ unsigned Fast = 0;
+ return TLI->allowsMisalignedMemoryAccessesImpl(
+ NewNumDwords * 32, AS, NewAlign,
+ Ld->getMemOperand()->getFlags(), &Fast) &&
+ Fast > 1;
+ }
+ case AMDGPUAS::PRIVATE_ADDRESS:
+ if (Subtarget->getMaxPrivateElementSize() < NewNumDwords * 4)
+ return false;
+
+ return TLI->allowsMisalignedMemoryAccessesImpl(
+ NewNumDwords * 32, AS, NewAlign,
+ Ld->getMemOperand()->getFlags());
+ default:
+ return false;
+ }
+}
+
+// Shrink a dword vector load when the used lanes form a contiguous range.
+static SDValue shrinkExtractedDwordLoad(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const GCNSubtarget *Subtarget,
+ const SITargetLowering *TLI) {
+ if (!DCI.isAfterLegalizeDAG())
+ return SDValue();
+
+ SDValue Vec = N->getOperand(0);
+ auto *Ld = dyn_cast<LoadSDNode>(Vec);
+ MVT OldVT = Vec.getSimpleValueType();
+ if (!Ld || !OldVT.isVector() || OldVT.getVectorElementType() != MVT::i32 ||
+ !Ld->isSimple() || !ISD::isNormalLoad(Ld))
+ return SDValue();
+
+ unsigned OldNumLanes = OldVT.getVectorNumElements();
+ APInt UsedLanes = APInt::getZero(OldNumLanes);
+ SmallVector<SDNode *, 4> Extracts;
+ for (SDUse &Use : Ld->uses()) {
+ if (Use.getResNo() != 0)
+ continue;
+
+ SDNode *User = Use.getUser();
+ if (User->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return SDValue();
+
+ auto *Idx = dyn_cast<ConstantSDNode>(User->getOperand(1));
+ if (!Idx)
+ return SDValue();
+
+ uint64_t Lane = Idx->getZExtValue();
+ if (Lane >= OldNumLanes)
+ return SDValue();
+
+ UsedLanes.setBit(Lane);
+ Extracts.push_back(User);
+ }
+
+ if (Extracts.empty())
+ return SDValue();
+
+ // The used lane mask must be one contiguous run. This also returns the start
+ // lane and run length for the replacement load.
+ unsigned FirstLane = 0;
+ unsigned NewNumLanes = 0;
+ if (!UsedLanes.isShiftedMask(FirstLane, NewNumLanes))
+ return SDValue();
+
+ if (NewNumLanes >= OldNumLanes)
+ return SDValue();
+
+ uint64_t ByteOffset = FirstLane * 4;
+ if (!canShrinkDwordVectorLoad(Ld, Subtarget, TLI, NewNumLanes, ByteOffset))
+ return SDValue();
+
+ SDLoc SL(Ld);
+ SelectionDAG &DAG = DCI.DAG;
+ MachineFunction &MF = DAG.getMachineFunction();
+ MVT NewVT = MVT::getVectorVT(MVT::i32, NewNumLanes);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ Ld->getMemOperand(), ByteOffset, NewVT.getStoreSize());
+ SDValue NewBase =
+ ByteOffset == 0
+ ? Ld->getBasePtr()
+ : DAG.getMemBasePlusOffset(Ld->getBasePtr(),
+ TypeSize::getFixed(ByteOffset), SL);
+ SDValue NewLoad = DAG.getLoad(NewVT, SL, Ld->getChain(), NewBase, MMO);
+ DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
+ DCI.AddToWorklist(NewLoad.getNode());
+
+ for (SDNode *Extract : Extracts) {
+ if (Extract == N)
+ continue;
+
+ unsigned Lane =
+ cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue();
+ SDValue NewIdx =
+ DAG.getVectorIdxConstant(Lane - FirstLane, SDLoc(Extract));
+ SDNode *NewExtract = DAG.UpdateNodeOperands(Extract, NewLoad, NewIdx);
+ DCI.AddToWorklist(NewExtract);
+ if (NewExtract != Extract) {
+ DAG.ReplaceAllUsesWith(SDValue(Extract, 0), SDValue(NewExtract, 0));
+ DAG.RemoveDeadNode(Extract);
+ }
+ }
+
+ auto *Idx = cast<ConstantSDNode>(N->getOperand(1));
+ SDValue NewIdx =
+ DAG.getVectorIdxConstant(Idx->getZExtValue() - FirstLane, SDLoc(N));
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
+ NewLoad, NewIdx);
+}
+
SDValue
SITargetLowering::performExtractVectorEltCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -16628,8 +16793,11 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
}
}
- if (!DCI.isBeforeLegalize())
+ if (!DCI.isBeforeLegalize()) {
+ if (SDValue Shrunk = shrinkExtractedDwordLoad(N, DCI, Subtarget, this))
+ return Shrunk;
return SDValue();
+ }
// Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
// elements. This exposes more load reduction opportunities by replacing
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 2ac5574be0def..8b8e7ed21815e 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -5897,7 +5897,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX8-LABEL: global_extload_v5bf16_to_v5f32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx4 v[2:5], v[0:1]
+; GFX8-NEXT: flat_load_dwordx3 v[2:4], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5909,7 +5909,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX9-LABEL: global_extload_v5bf16_to_v5f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
+; GFX9-NEXT: global_load_dwordx3 v[2:4], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5921,7 +5921,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX10-LABEL: global_extload_v5bf16_to_v5f32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
+; GFX10-NEXT: global_load_dwordx3 v[2:4], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5933,7 +5933,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX11-LABEL: global_extload_v5bf16_to_v5f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX11-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5946,7 +5946,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX1250-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2
@@ -7125,7 +7125,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX8-LABEL: global_extload_v5bf16_to_v5f64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT: flat_load_dwordx3 v[0:2], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
@@ -7142,7 +7142,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX9-LABEL: global_extload_v5bf16_to_v5f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT: global_load_dwordx3 v[0:2], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
@@ -7159,7 +7159,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX10-LABEL: global_extload_v5bf16_to_v5f64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
+; GFX10-NEXT: global_load_dwordx3 v[2:4], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -7176,7 +7176,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX11-LABEL: global_extload_v5bf16_to_v5f64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX11-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -7194,7 +7194,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX1250-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_lshlrev_b32 v5, 16, v3
diff --git a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
index 3126491559d0a..a7e5949e5501a 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
@@ -4,8 +4,8 @@
; This caused failure in infinite cycle in Selection DAG (combine) due to missing insert_subvector.
;
; CHECK-LABEL: name: test1
-; CHECK: GLOBAL_LOAD_DWORDX4
-; CHECK: GLOBAL_LOAD_DWORDX4
+; CHECK: GLOBAL_LOAD_DWORDX2
+; CHECK: GLOBAL_LOAD_DWORDX2
; CHECK: GLOBAL_STORE_DWORDX4
define protected amdgpu_kernel void @test1(ptr addrspace(4) %ptr) local_unnamed_addr !kernel_arg_addr_space !0 !kernel_arg_access_qual !1 !kernel_arg_type !2 !kernel_arg_base_type !2 !kernel_arg_type_qual !3 !kernel_arg_name !4 {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 57fe32256e624..a0792a3252cfa 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -506,7 +506,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx3 v[48:50], v[4:5]
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
@@ -544,7 +544,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx3 v[48:50], v[4:5]
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
@@ -570,30 +570,29 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX10-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX10-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT: v_add_co_u32 v32, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, 0x8c, v0
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
; GFX10-SDAG-NEXT: s_clause 0x8
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[36:37] offset:80
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[36:37] offset:96
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[36:37] offset:48
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[36:37] offset:64
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[36:37] offset:16
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[36:37] offset:32
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[32:33] offset:48
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[32:33] offset:96
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[32:33] offset:64
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[32:33] offset:80
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[32:33] offset:16
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[32:33] offset:32
; GFX10-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:12
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[36:37]
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[36:39], v[36:37] offset:112
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[32:33]
+; GFX10-SDAG-NEXT: flat_load_dwordx3 v[36:38], v[36:37]
; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX10-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(8) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(5)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:48
; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11] offset:64
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7] offset:64
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[12:15] offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
@@ -638,7 +637,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[52:55], v[0:1] offset:12
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x8c
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 a[0:3], v[0:1]
+; GFX942-SDAG-NEXT: flat_load_dwordx3 a[0:2], v[0:1]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x60
; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x50
@@ -671,38 +670,45 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX11-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX11-SDAG-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX11-SDAG-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX11-SDAG-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX11-SDAG-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX11-SDAG-NEXT: flat_load_b128 v[24:27], v[36:37]
+; GFX11-SDAG-NEXT: v_add_co_u32 v32, vcc_lo, v0, 28
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT: v_add_co_u32 v34, vcc_lo, 0x8c, v0
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v35, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT: s_clause 0x8
+; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[32:33] offset:96
+; GFX11-SDAG-NEXT: flat_load_b128 v[8:11], v[32:33] offset:64
+; GFX11-SDAG-NEXT: flat_load_b128 v[12:15], v[32:33] offset:80
+; GFX11-SDAG-NEXT: flat_load_b128 v[16:19], v[32:33] offset:32
+; GFX11-SDAG-NEXT: flat_load_b128 v[20:23], v[32:33] offset:16
+; GFX11-SDAG-NEXT: flat_load_b128 v[24:27], v[32:33]
; GFX11-SDAG-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX11-SDAG-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
+; GFX11-SDAG-NEXT: flat_load_b96 v[36:38], v[34:35]
+; GFX11-SDAG-NEXT: flat_load_b128 v[32:35], v[32:33] offset:48
; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX11-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(6) lgkmcnt(6)
+; GFX11-SDAG-NEXT: s_clause 0x2
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:48
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:64
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:32
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[16:19]
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(2) lgkmcnt(8)
; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[28:31]
-; GFX11-SDAG-NEXT: flat_store_b32 v[48:49], v34
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(1) lgkmcnt(8)
+; GFX11-SDAG-NEXT: flat_store_b32 v[48:49], v38
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(8)
; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
-; GFX11-SDAG-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
+; GFX11-SDAG-NEXT: flat_store_b128 v[2:3], v[32:35] offset:64
+; GFX11-SDAG-NEXT: flat_store_b64 v[2:3], v[36:37] offset:128
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -720,40 +726,48 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX12-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX12-SDAG-NEXT: v_add_co_u32 v32, vcc_lo, v0, 0x7c
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX12-SDAG-NEXT: s_clause 0x7
-; GFX12-SDAG-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX12-SDAG-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX12-SDAG-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX12-SDAG-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX12-SDAG-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX12-SDAG-NEXT: flat_load_b128 v[24:27], v[36:37]
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v1, vcc_lo
+; GFX12-SDAG-NEXT: v_add_co_u32 v34, vcc_lo, 0x8c, v0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v35, null, 0, v1, vcc_lo
+; GFX12-SDAG-NEXT: s_clause 0x8
+; GFX12-SDAG-NEXT: flat_load_b128 v[4:7], v[32:33]
+; GFX12-SDAG-NEXT: flat_load_b128 v[8:11], v[32:33] offset:-32
+; GFX12-SDAG-NEXT: flat_load_b128 v[12:15], v[32:33] offset:-16
+; GFX12-SDAG-NEXT: flat_load_b128 v[16:19], v[32:33] offset:-64
+; GFX12-SDAG-NEXT: flat_load_b128 v[20:23], v[32:33] offset:-80
+; GFX12-SDAG-NEXT: flat_load_b128 v[24:27], v[32:33] offset:-96
; GFX12-SDAG-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX12-SDAG-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
-; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-SDAG-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
+; GFX12-SDAG-NEXT: flat_load_b96 v[36:38], v[34:35]
+; GFX12-SDAG-NEXT: flat_load_b128 v[32:35], v[32:33] offset:-48
; GFX12-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX12-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX12-SDAG-NEXT: s_clause 0x7
-; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x606
+; GFX12-SDAG-NEXT: s_clause 0x2
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:48
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:64
+; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:32
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x508
; GFX12-SDAG-NEXT: flat_store_b128 v[0:1], v[16:19]
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x408
; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[20:23] offset:32
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x308
; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[24:27] offset:16
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x208
; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[28:31]
-; GFX12-SDAG-NEXT: flat_store_b32 v[48:49], v34
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x108
+; GFX12-SDAG-NEXT: flat_store_b32 v[48:49], v38
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x8
; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[35:38] offset:64
-; GFX12-SDAG-NEXT: flat_store_b64 v[2:3], v[32:33] offset:128
+; GFX12-SDAG-NEXT: flat_store_b128 v[2:3], v[32:35] offset:64
+; GFX12-SDAG-NEXT: flat_store_b64 v[2:3], v[36:37] offset:128
; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index a7665ee31e608..906e4c86c67bb 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -999,7 +999,7 @@ define <5 x i16> @v5i16_func_void() #0 {
; GFX89-NEXT: s_mov_b32 s7, 0xf000
; GFX89-NEXT: s_mov_b32 s6, -1
; GFX89-NEXT: s_waitcnt lgkmcnt(0)
-; GFX89-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
+; GFX89-NEXT: buffer_load_dwordx3 v[0:2], off, s[4:7], 0
; GFX89-NEXT: s_waitcnt vmcnt(0)
; GFX89-NEXT: s_setpc_b64 s[30:31]
;
@@ -1010,7 +1010,7 @@ define <5 x i16> @v5i16_func_void() #0 {
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0
+; GFX11-NEXT: buffer_load_b96 v[0:2], off, s[0:3], 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: s_setpc_b64 s[30:31]
%ptr = load volatile ptr addrspace(1), ptr addrspace(4) poison
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
index 04f7c3583bd73..2bdb945b8cfc4 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll
@@ -3223,7 +3223,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s33, s33, 0xfffffe00
; GFX11-NEXT: s_or_saveexec_b32 s0, -1
-; GFX11-NEXT: scratch_store_b32 off, v62, s33 offset:1600 ; 4-byte Folded Spill
+; GFX11-NEXT: scratch_store_b32 off, v62, s33 offset:1596 ; 4-byte Folded Spill
; GFX11-NEXT: s_mov_b32 exec_lo, s0
; GFX11-NEXT: s_mov_b32 s39, s34
; GFX11-NEXT: s_mov_b32 s34, s32
@@ -3319,7 +3319,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: scratch_load_b128 v[36:39], off, s33 offset:704
; GFX11-NEXT: scratch_load_b128 v[32:35], off, s33 offset:688
; GFX11-NEXT: scratch_load_b128 v[28:31], off, s33 offset:672
-; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:512
+; GFX11-NEXT: scratch_load_b96 v[20:22], off, s33 offset:516
; GFX11-NEXT: s_waitcnt vmcnt(6)
; GFX11-NEXT: v_mov_b32_e32 v6, 24
; GFX11-NEXT: s_add_i32 s2, s32, 0xa0
@@ -3332,17 +3332,17 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v42, v2
; GFX11-NEXT: v_mov_b32_e32 v51, v24
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1584 ; 16-byte Folded Spill
+; GFX11-NEXT: scratch_store_b96 off, v[20:22], s33 offset:1552 ; 12-byte Folded Spill
; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:528
-; GFX11-NEXT: v_mov_b32_e32 v52, v25
+; GFX11-NEXT: v_dual_mov_b32 v52, v25 :: v_dual_mov_b32 v1, 42
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1536 ; 16-byte Folded Spill
+; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1580 ; 16-byte Folded Spill
; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:544
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1568 ; 16-byte Folded Spill
+; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1536 ; 16-byte Folded Spill
; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:560
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1552 ; 16-byte Folded Spill
+; GFX11-NEXT: scratch_store_b128 off, v[20:23], s33 offset:1564 ; 16-byte Folded Spill
; GFX11-NEXT: s_clause 0x2
; GFX11-NEXT: scratch_load_b128 v[20:23], off, s33 offset:576
; GFX11-NEXT: scratch_load_b128 v[58:61], off, s33 offset:592
@@ -3366,21 +3366,23 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: s_add_i32 s2, s32, 16
; GFX11-NEXT: v_mov_b32_e32 v29, v43
; GFX11-NEXT: scratch_store_b128 off, v[50:53], s2
-; GFX11-NEXT: s_clause 0x3 ; 64-byte Folded Reload
-; GFX11-NEXT: scratch_load_b128 v[1:4], off, s33 offset:1584
-; GFX11-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1536
-; GFX11-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1568
-; GFX11-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1552
+; GFX11-NEXT: s_clause 0x3 ; 60-byte Folded Reload
+; GFX11-NEXT: scratch_load_b128 v[5:8], off, s33 offset:1580
+; GFX11-NEXT: scratch_load_b128 v[9:12], off, s33 offset:1536
+; GFX11-NEXT: scratch_load_b128 v[13:16], off, s33 offset:1564
+; GFX11-NEXT: scratch_load_b96 v[2:4], off, s33 offset:1552
; GFX11-NEXT: s_add_i32 s2, s33, 0x400
; GFX11-NEXT: v_dual_mov_b32 v30, v44 :: v_dual_mov_b32 v31, v45
-; GFX11-NEXT: s_waitcnt vmcnt(3)
-; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, 42
-; GFX11-NEXT: v_dual_mov_b32 v17, v20 :: v_dual_mov_b32 v18, v21
-; GFX11-NEXT: v_dual_mov_b32 v19, v22 :: v_dual_mov_b32 v20, v23
-; GFX11-NEXT: v_dual_mov_b32 v21, v58 :: v_dual_mov_b32 v22, v59
-; GFX11-NEXT: v_dual_mov_b32 v23, v60 :: v_dual_mov_b32 v24, v61
-; GFX11-NEXT: v_dual_mov_b32 v25, v54 :: v_dual_mov_b32 v26, v55
-; GFX11-NEXT: v_dual_mov_b32 v27, v56 :: v_dual_mov_b32 v28, v57
+; GFX11-NEXT: s_waitcnt vmcnt(6)
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v17, v20
+; GFX11-NEXT: v_dual_mov_b32 v18, v21 :: v_dual_mov_b32 v19, v22
+; GFX11-NEXT: s_waitcnt vmcnt(5)
+; GFX11-NEXT: v_dual_mov_b32 v20, v23 :: v_dual_mov_b32 v21, v58
+; GFX11-NEXT: v_dual_mov_b32 v22, v59 :: v_dual_mov_b32 v23, v60
+; GFX11-NEXT: s_waitcnt vmcnt(4)
+; GFX11-NEXT: v_dual_mov_b32 v24, v61 :: v_dual_mov_b32 v25, v54
+; GFX11-NEXT: v_dual_mov_b32 v26, v55 :: v_dual_mov_b32 v27, v56
+; GFX11-NEXT: v_mov_b32_e32 v28, v57
; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-NEXT: s_clause 0xd ; 56-byte Folded Reload
; GFX11-NEXT: scratch_load_b32 v61, off, s33
@@ -3402,7 +3404,7 @@ define amdgpu_gfx void @call_72xi32() #1 {
; GFX11-NEXT: s_mov_b32 s32, s34
; GFX11-NEXT: s_mov_b32 s34, s39
; GFX11-NEXT: s_or_saveexec_b32 s0, -1
-; GFX11-NEXT: scratch_load_b32 v62, off, s33 offset:1600 ; 4-byte Folded Reload
+; GFX11-NEXT: scratch_load_b32 v62, off, s33 offset:1596 ; 4-byte Folded Reload
; GFX11-NEXT: s_mov_b32 exec_lo, s0
; GFX11-NEXT: s_mov_b32 s33, s38
; GFX11-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index b67c40b2d8333..5696bcc62dd90 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -5672,31 +5672,30 @@ define amdgpu_kernel void @idot4_acc32_v16i8(ptr addrspace(1) %src1,
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s10, 0
; GFX7-NEXT: s_mov_b32 s11, s7
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v0
; GFX7-NEXT: v_mov_b32_e32 v2, 0
-; GFX7-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]
; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, v2
-; GFX7-NEXT: buffer_load_dwordx4 v[0:3], v[1:2], s[8:11], 0 addr64
-; GFX7-NEXT: buffer_load_dword v0, v[4:5], s[0:3], 0 addr64
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[1:2], s[0:3], 0 addr64 offset:8
+; GFX7-NEXT: buffer_load_dword v2, v[3:4], s[8:11], 0 addr64
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
+; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 8
-; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX7-NEXT: v_mul_u32_u24_e32 v2, v2, v5
-; GFX7-NEXT: v_bfe_u32 v6, v3, 8, 8
-; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
-; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
-; GFX7-NEXT: v_mad_u32_u24 v1, v6, v7, v1
-; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
+; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 8
+; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v2
+; GFX7-NEXT: v_mul_u32_u24_e32 v0, v0, v5
+; GFX7-NEXT: v_bfe_u32 v6, v1, 8, 8
+; GFX7-NEXT: v_bfe_u32 v7, v2, 16, 8
+; GFX7-NEXT: v_mad_u32_u24 v0, v3, v4, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX7-NEXT: v_mad_u32_u24 v0, v6, v7, v0
+; GFX7-NEXT: v_mad_u32_u24 v0, v1, v2, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
@@ -5708,72 +5707,66 @@ define amdgpu_kernel void @idot4_acc32_v16i8(ptr addrspace(1) %src1,
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v1
-; GFX8-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, s2, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc
-; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[1:2]
-; GFX8-NEXT: flat_load_dword v4, v[4:5]
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, s0, v1
+; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v2, vcc
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 8, v3
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mul_u32_u24_sdwa v5, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_mul_u32_u24_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_1
-; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_2
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v2
-; GFX8-NEXT: v_mul_u32_u24_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: v_mul_u32_u24_sdwa v5, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_1
+; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v5, v0
+; GFX8-NEXT: v_mul_u32_u24_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v6
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
+; GFX8-NEXT: flat_store_dword v[2:3], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-NODL-LABEL: idot4_acc32_v16i8:
; GFX9-NODL: ; %bb.0: ; %entry
; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 4, v0
-; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v5, 3, v0
-; GFX9-NODL-NEXT: ; kill: killed $vgpr4
-; GFX9-NODL-NEXT: ; kill: killed $vgpr5
-; GFX9-NODL-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2 killed $sgpr3
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v2, 4, v0
+; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT: global_load_dwordx4 v[0:3], v4, s[0:1]
-; GFX9-NODL-NEXT: global_load_dword v0, v5, s[2:3]
-; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] offset:8
+; GFX9-NODL-NEXT: global_load_dword v4, v3, s[2:3]
+; GFX9-NODL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_1
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_2
-; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NODL-NEXT: v_add_u32_e32 v2, v4, v2
-; GFX9-NODL-NEXT: v_add3_u32 v0, v2, v5, v0
-; GFX9-NODL-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_1
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_2
+; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NODL-NEXT: v_add_u32_e32 v0, v3, v0
+; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v5, v1
+; GFX9-NODL-NEXT: global_store_dword v2, v0, s[6:7]
; GFX9-NODL-NEXT: s_endpgm
;
; GFX9-DL-LABEL: idot4_acc32_v16i8:
; GFX9-DL: ; %bb.0: ; %entry
; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 4, v0
-; GFX9-DL-NEXT: v_lshlrev_b32_e32 v5, 3, v0
-; GFX9-DL-NEXT: ; kill: killed $vgpr4
-; GFX9-DL-NEXT: ; kill: killed $vgpr5
-; GFX9-DL-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2 killed $sgpr3
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v2, 4, v0
+; GFX9-DL-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT: global_load_dwordx4 v[0:3], v4, s[0:1]
-; GFX9-DL-NEXT: global_load_dword v0, v5, s[2:3]
+; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] offset:8
+; GFX9-DL-NEXT: global_load_dword v4, v3, s[2:3]
; GFX9-DL-NEXT: s_mov_b32 s0, 0x7050002
; GFX9-DL-NEXT: s_mov_b32 s1, 0x3020001
+; GFX9-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-DL-NEXT: v_perm_b32 v2, v3, v2, s0
+; GFX9-DL-NEXT: v_perm_b32 v0, v1, v0, s0
; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT: v_perm_b32 v0, v0, v0, s1
-; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v0, 0
-; GFX9-DL-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX9-DL-NEXT: v_perm_b32 v1, v4, v4, s1
+; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
+; GFX9-DL-NEXT: global_store_dword v2, v0, s[6:7]
; GFX9-DL-NEXT: s_endpgm
;
; GFX10-DL-LABEL: idot4_acc32_v16i8:
@@ -5781,20 +5774,17 @@ define amdgpu_kernel void @idot4_acc32_v16i8(ptr addrspace(1) %src1,
; GFX10-DL-NEXT: s_clause 0x1
; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 4, v0
-; GFX10-DL-NEXT: v_lshlrev_b32_e32 v5, 3, v0
-; GFX10-DL-NEXT: ; kill: killed $vgpr4
-; GFX10-DL-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2 killed $sgpr3
-; GFX10-DL-NEXT: ; kill: killed $vgpr5
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v2, 4, v0
+; GFX10-DL-NEXT: v_lshlrev_b32_e32 v3, 3, v0
; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DL-NEXT: global_load_dwordx4 v[0:3], v4, s[0:1]
-; GFX10-DL-NEXT: global_load_dword v0, v5, s[2:3]
+; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] offset:8
+; GFX10-DL-NEXT: global_load_dword v4, v3, s[2:3]
+; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x7050002
+; GFX10-DL-NEXT: v_perm_b32 v0, v1, v0, 0x7050002
; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT: v_perm_b32 v0, v0, v0, 0x3020001
-; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0
+; GFX10-DL-NEXT: v_perm_b32 v1, v4, v4, 0x3020001
+; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]
; GFX10-DL-NEXT: s_endpgm
;
@@ -5805,17 +5795,17 @@ define amdgpu_kernel void @idot4_acc32_v16i8(ptr addrspace(1) %src1,
; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-NEXT: v_lshlrev_b32_e32 v1, 4, v0
-; GFX11-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v0
+; GFX11-DL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-DL-NEXT: global_load_b128 v[0:3], v1, s[0:1]
-; GFX11-DL-NEXT: global_load_b32 v0, v4, s[2:3]
+; GFX11-DL-NEXT: global_load_b64 v[0:1], v1, s[0:1] offset:8
+; GFX11-DL-NEXT: global_load_b32 v2, v2, s[2:3]
; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-NEXT: v_perm_b32 v1, v3, v2, 0x7050002
+; GFX11-DL-NEXT: v_perm_b32 v0, v1, v0, 0x7050002
; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0x3020001
+; GFX11-DL-NEXT: v_perm_b32 v1, v2, v2, 0x3020001
; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0
+; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 0
; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]
; GFX11-DL-NEXT: s_endpgm
ptr addrspace(1) %src2,
diff --git a/llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll b/llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll
index 75c5d206e7933..9e6f617840a16 100644
--- a/llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll
@@ -61,36 +61,34 @@ define <3 x float> @extract_subvector_v3f32_v33f32_elt30_1(ptr addrspace(1) %ptr
; GFX900-LABEL: extract_subvector_v3f32_v33f32_elt30_1:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX900-NEXT: global_load_dwordx4 v[7:10], v[0:1], off offset:112
-; GFX900-NEXT: global_load_dword v2, v[0:1], off offset:128
+; GFX900-NEXT: v_mov_b32_e32 v4, v1
+; GFX900-NEXT: v_mov_b32_e32 v3, v0
+; GFX900-NEXT: global_load_dwordx4 v[5:8], v[3:4], off
+; GFX900-NEXT: global_load_dwordx3 v[0:2], v[3:4], off offset:120
; GFX900-NEXT: s_mov_b32 s4, 0
; GFX900-NEXT: s_mov_b32 s5, s4
; GFX900-NEXT: s_mov_b32 s6, s4
; GFX900-NEXT: s_mov_b32 s7, s4
-; GFX900-NEXT: s_waitcnt vmcnt(2)
-; GFX900-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0
-; GFX900-NEXT: s_waitcnt vmcnt(2)
-; GFX900-NEXT: v_mov_b32_e32 v0, v9
-; GFX900-NEXT: v_mov_b32_e32 v1, v10
+; GFX900-NEXT: s_waitcnt vmcnt(1)
+; GFX900-NEXT: buffer_store_dwordx4 v[5:8], off, s[4:7], 0
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: extract_subvector_v3f32_v33f32_elt30_1:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX942-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:112
-; GFX942-NEXT: global_load_dword v2, v[0:1], off offset:128
+; GFX942-NEXT: global_load_dwordx4 v[6:9], v[0:1], off
+; GFX942-NEXT: global_load_dwordx3 v[2:4], v[0:1], off offset:120
; GFX942-NEXT: s_mov_b32 s0, 0
; GFX942-NEXT: s_mov_b32 s1, s0
; GFX942-NEXT: s_mov_b32 s2, s0
; GFX942-NEXT: s_mov_b32 s3, s0
-; GFX942-NEXT: s_waitcnt vmcnt(2)
-; GFX942-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
-; GFX942-NEXT: s_waitcnt vmcnt(2)
-; GFX942-NEXT: v_mov_b32_e32 v0, v10
-; GFX942-NEXT: v_mov_b32_e32 v1, v11
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0
+; GFX942-NEXT: s_waitcnt vmcnt(1)
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v1, v3
+; GFX942-NEXT: v_mov_b32_e32 v2, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = load <33 x float>, ptr addrspace(1) %ptr, align 4
@@ -104,36 +102,36 @@ define <6 x float> @extract_subvector_v6f32_v36f32_elt30(ptr addrspace(1) %ptr)
; GFX900-LABEL: extract_subvector_v6f32_v36f32_elt30:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: global_load_dwordx4 v[6:9], v[0:1], off
-; GFX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:112
+; GFX900-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
+; GFX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:120
; GFX900-NEXT: global_load_dwordx4 v[2:5], v[0:1], off offset:128
; GFX900-NEXT: s_mov_b32 s4, 0
; GFX900-NEXT: s_mov_b32 s5, s4
; GFX900-NEXT: s_mov_b32 s6, s4
; GFX900-NEXT: s_mov_b32 s7, s4
; GFX900-NEXT: s_waitcnt vmcnt(2)
-; GFX900-NEXT: buffer_store_dwordx4 v[6:9], off, s[4:7], 0
+; GFX900-NEXT: buffer_store_dwordx4 v[8:11], off, s[4:7], 0
; GFX900-NEXT: s_waitcnt vmcnt(2)
-; GFX900-NEXT: v_mov_b32_e32 v0, v12
-; GFX900-NEXT: v_mov_b32_e32 v1, v13
+; GFX900-NEXT: v_mov_b32_e32 v0, v6
+; GFX900-NEXT: v_mov_b32_e32 v1, v7
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: extract_subvector_v6f32_v36f32_elt30:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx4 v[6:9], v[0:1], off
-; GFX942-NEXT: global_load_dwordx4 v[10:13], v[0:1], off offset:112
+; GFX942-NEXT: global_load_dwordx4 v[8:11], v[0:1], off
+; GFX942-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:120
; GFX942-NEXT: global_load_dwordx4 v[2:5], v[0:1], off offset:128
; GFX942-NEXT: s_mov_b32 s0, 0
; GFX942-NEXT: s_mov_b32 s1, s0
; GFX942-NEXT: s_mov_b32 s2, s0
; GFX942-NEXT: s_mov_b32 s3, s0
; GFX942-NEXT: s_waitcnt vmcnt(2)
-; GFX942-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0
+; GFX942-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
; GFX942-NEXT: s_waitcnt vmcnt(2)
-; GFX942-NEXT: v_mov_b32_e32 v0, v12
-; GFX942-NEXT: v_mov_b32_e32 v1, v13
+; GFX942-NEXT: v_mov_b32_e32 v0, v6
+; GFX942-NEXT: v_mov_b32_e32 v1, v7
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = load <36 x float>, ptr addrspace(1) %ptr, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
index e245feb68f587..9a7f0799e5a88 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll
@@ -339,10 +339,10 @@ define amdgpu_kernel void @test_readlane_vregs_i64(ptr addrspace(1) %out, ptr ad
; CHECK-SDAG-NEXT: v_mov_b32_e32 v1, s3
; CHECK-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; CHECK-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-SDAG-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; CHECK-SDAG-NEXT: s_waitcnt vmcnt(0)
+; CHECK-SDAG-NEXT: flat_load_dwordx3 v[0:2], v[0:1]
; CHECK-SDAG-NEXT: v_mov_b32_e32 v3, s0
; CHECK-SDAG-NEXT: v_mov_b32_e32 v4, s1
+; CHECK-SDAG-NEXT: s_waitcnt vmcnt(0)
; CHECK-SDAG-NEXT: v_readfirstlane_b32 s0, v2
; CHECK-SDAG-NEXT: s_nop 3
; CHECK-SDAG-NEXT: v_readlane_b32 s1, v1, s0
@@ -399,10 +399,10 @@ define amdgpu_kernel void @test_readlane_vregs_f64(ptr addrspace(1) %out, ptr ad
; CHECK-SDAG-NEXT: v_mov_b32_e32 v1, s3
; CHECK-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; CHECK-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; CHECK-SDAG-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; CHECK-SDAG-NEXT: s_waitcnt vmcnt(0)
+; CHECK-SDAG-NEXT: flat_load_dwordx3 v[0:2], v[0:1]
; CHECK-SDAG-NEXT: v_mov_b32_e32 v3, s0
; CHECK-SDAG-NEXT: v_mov_b32_e32 v4, s1
+; CHECK-SDAG-NEXT: s_waitcnt vmcnt(0)
; CHECK-SDAG-NEXT: v_readfirstlane_b32 s0, v2
; CHECK-SDAG-NEXT: s_nop 3
; CHECK-SDAG-NEXT: v_readlane_b32 s1, v1, s0
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index b54d80a03abd0..5f50c604d7385 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -3777,13 +3777,13 @@ define hidden void @extract_v13i64(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_clause 0x2
; GFX10-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
-; GFX10-NEXT: global_load_dwordx4 v[11:14], v[0:1], off
-; GFX10-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:64
+; GFX10-NEXT: global_load_dwordx2 v[2:3], v[0:1], off offset:4
+; GFX10-NEXT: global_load_dwordx4 v[11:14], v[0:1], off offset:64
; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_perm_b32 v0, v12, v13, 0x1000504
+; GFX10-NEXT: v_perm_b32 v0, v2, v3, 0x1000504
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_perm_b32 v1, v10, v14, 0x1000504
+; GFX10-NEXT: v_perm_b32 v1, v10, v11, 0x1000504
; GFX10-NEXT: global_store_dword v[4:5], v0, off
; GFX10-NEXT: global_store_dword v[6:7], v1, off
; GFX10-NEXT: s_setpc_b64 s[30:31]
@@ -3792,14 +3792,14 @@ define hidden void @extract_v13i64(ptr addrspace(1) %in0, ptr addrspace(1) %in1,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48
-; GFX9-NEXT: global_load_dwordx4 v[11:14], v[0:1], off
-; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:64
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v[0:1], off offset:4
+; GFX9-NEXT: global_load_dwordx4 v[11:14], v[0:1], off offset:64
; GFX9-NEXT: s_mov_b32 s4, 0x1000504
; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_perm_b32 v0, v12, v13, s4
+; GFX9-NEXT: v_perm_b32 v0, v2, v3, s4
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_perm_b32 v1, v10, v14, s4
+; GFX9-NEXT: v_perm_b32 v1, v10, v11, s4
; GFX9-NEXT: global_store_dword v[4:5], v0, off
; GFX9-NEXT: global_store_dword v[6:7], v1, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-extracted-dword-load.ll b/llvm/test/CodeGen/AMDGPU/shrink-extracted-dword-load.ll
new file mode 100644
index 0000000000000..06c2dfd2f7fc1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/shrink-extracted-dword-load.ll
@@ -0,0 +1,357 @@
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1201 -stop-after=finalize-isel -o - < %s | FileCheck -check-prefix=ISEL %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1201 < %s | FileCheck -check-prefix=ISA %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-scalarize-global-loads -stop-after=finalize-isel -o - < %s | FileCheck -check-prefix=GFX11-SCALAR %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-scalarize-global-loads=false -stop-after=finalize-isel -o - < %s | FileCheck -check-prefix=GFX11-SCALAR %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -mattr=+max-private-element-size-8 -stop-after=finalize-isel -o - < %s | FileCheck -check-prefix=PRIVATE8 %s
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+define amdgpu_kernel void @shrink_i64_load_to_i32_chunks(ptr addrspace(1) %out,
+ ptr addrspace(4) %p) {
+; ISEL-LABEL: name: shrink_i64_load_to_i32_chunks
+; ISEL: GLOBAL_LOAD_DWORDX3{{.*}}load (s96)
+; ISEL: GLOBAL_LOAD_DWORDX3{{.*}}load (s96)
+; ISEL: GLOBAL_LOAD_DWORDX3{{.*}}load (s96)
+; ISEL-NOT: GLOBAL_LOAD_DWORDX4
+;
+; ISA-LABEL: shrink_i64_load_to_i32_chunks:
+; ISA: global_load_b96
+; ISA: global_load_b96
+; ISA: global_load_b96
+; ISA-NOT: global_load_b128
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %byte.offset = shl i32 %tid, 6
+ %byte.offset64 = zext i32 %byte.offset to i64
+ %p.div = getelementptr i8, ptr addrspace(4) %p, i64 %byte.offset64
+ %v = load <6 x i64>, ptr addrspace(4) %p.div, align 8
+ %e0 = extractelement <6 x i64> %v, i32 0
+ %e1 = extractelement <6 x i64> %v, i32 1
+ %e2 = extractelement <6 x i64> %v, i32 2
+ %e3 = extractelement <6 x i64> %v, i32 3
+ %e4 = extractelement <6 x i64> %v, i32 4
+ %e5 = extractelement <6 x i64> %v, i32 5
+
+ %d0 = trunc i64 %e0 to i32
+ %e0hi = lshr i64 %e0, 32
+ %d1 = trunc i64 %e0hi to i32
+ %d2 = trunc i64 %e1 to i32
+
+ %d4 = trunc i64 %e2 to i32
+ %e2hi = lshr i64 %e2, 32
+ %d5 = trunc i64 %e2hi to i32
+ %d6 = trunc i64 %e3 to i32
+
+ %d8 = trunc i64 %e4 to i32
+ %e4hi = lshr i64 %e4, 32
+ %d9 = trunc i64 %e4hi to i32
+ %d10 = trunc i64 %e5 to i32
+
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store volatile i32 %d0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out0, i32 1
+ store volatile i32 %d1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out0, i32 2
+ store volatile i32 %d2, ptr addrspace(1) %out2, align 4
+ %out4 = getelementptr i32, ptr addrspace(1) %out0, i32 4
+ store volatile i32 %d4, ptr addrspace(1) %out4, align 4
+ %out5 = getelementptr i32, ptr addrspace(1) %out0, i32 5
+ store volatile i32 %d5, ptr addrspace(1) %out5, align 4
+ %out6 = getelementptr i32, ptr addrspace(1) %out0, i32 6
+ store volatile i32 %d6, ptr addrspace(1) %out6, align 4
+ %out8 = getelementptr i32, ptr addrspace(1) %out0, i32 8
+ store volatile i32 %d8, ptr addrspace(1) %out8, align 4
+ %out9 = getelementptr i32, ptr addrspace(1) %out0, i32 9
+ store volatile i32 %d9, ptr addrspace(1) %out9, align 4
+ %out10 = getelementptr i32, ptr addrspace(1) %out0, i32 10
+ store volatile i32 %d10, ptr addrspace(1) %out10, align 4
+ ret void
+}
+
+define amdgpu_kernel void @keep_i64_load_when_high_dword_used(ptr addrspace(1) %out,
+ ptr addrspace(4) %p) {
+; ISEL-LABEL: name: keep_i64_load_when_high_dword_used
+; ISEL: GLOBAL_LOAD_DWORDX4
+; ISEL-NOT: GLOBAL_LOAD_DWORDX3
+;
+; ISA-LABEL: keep_i64_load_when_high_dword_used:
+; ISA: global_load_b128
+; ISA-NOT: global_load_b96
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %byte.offset = shl i32 %tid, 4
+ %byte.offset64 = zext i32 %byte.offset to i64
+ %p.div = getelementptr i8, ptr addrspace(4) %p, i64 %byte.offset64
+ %v = load <2 x i64>, ptr addrspace(4) %p.div, align 8
+ %e0 = extractelement <2 x i64> %v, i32 0
+ %e1 = extractelement <2 x i64> %v, i32 1
+ %d0 = trunc i64 %e0 to i32
+ %e1hi = lshr i64 %e1, 32
+ %d3 = trunc i64 %e1hi to i32
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store volatile i32 %d0, ptr addrspace(1) %out0, align 4
+ %out3 = getelementptr i32, ptr addrspace(1) %out0, i32 3
+ store volatile i32 %d3, ptr addrspace(1) %out3, align 4
+ ret void
+}
+
+define amdgpu_kernel void @keep_non_contiguous_lanes(ptr addrspace(1) %out,
+ ptr addrspace(4) %p) {
+; ISEL-LABEL: name: keep_non_contiguous_lanes
+; ISEL: GLOBAL_LOAD_DWORDX4
+; ISEL-NOT: GLOBAL_LOAD_DWORDX2
+; ISEL-NOT: GLOBAL_LOAD_DWORDX3
+;
+; ISA-LABEL: keep_non_contiguous_lanes:
+; ISA: global_load_b128
+; ISA-NOT: global_load_b64
+; ISA-NOT: global_load_b96
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %byte.offset = shl i32 %tid, 4
+ %byte.offset64 = zext i32 %byte.offset to i64
+ %p.div = getelementptr i8, ptr addrspace(4) %p, i64 %byte.offset64
+ %v = load <4 x i32>, ptr addrspace(4) %p.div, align 8
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e2 = extractelement <4 x i32> %v, i32 2
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store volatile i32 %e0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out0, i32 1
+ store volatile i32 %e2, ptr addrspace(1) %out1, align 4
+ ret void
+}
+
+define amdgpu_kernel void @shrink_v3i32_load_to_v2i32(ptr addrspace(1) %out,
+ ptr addrspace(4) %p) {
+; ISEL-LABEL: name: shrink_v3i32_load_to_v2i32
+; ISEL: GLOBAL_LOAD_DWORDX2{{.*}}load (s64)
+; ISEL-NOT: GLOBAL_LOAD_DWORDX3
+;
+; ISA-LABEL: shrink_v3i32_load_to_v2i32:
+; ISA: global_load_b64
+; ISA-NOT: global_load_b96
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %byte.offset = shl i32 %tid, 4
+ %byte.offset64 = zext i32 %byte.offset to i64
+ %p.div = getelementptr i8, ptr addrspace(4) %p, i64 %byte.offset64
+ %v = load <3 x i32>, ptr addrspace(4) %p.div, align 8
+ %e0 = extractelement <3 x i32> %v, i32 0
+ %e1 = extractelement <3 x i32> %v, i32 1
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store volatile i32 %e0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out0, i32 1
+ store volatile i32 %e1, ptr addrspace(1) %out1, align 4
+ ret void
+}
+
+define amdgpu_kernel void @shrink_v4i32_middle_to_v2i32(ptr addrspace(1) %out,
+ ptr addrspace(4) %p) {
+; ISEL-LABEL: name: shrink_v4i32_middle_to_v2i32
+; ISEL: GLOBAL_LOAD_DWORDX2{{.*}}load (s64) from %ir.p.div + 4
+; ISEL-NOT: GLOBAL_LOAD_DWORDX4
+;
+; ISA-LABEL: shrink_v4i32_middle_to_v2i32:
+; ISA: global_load_b64
+; ISA-NOT: global_load_b128
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %byte.offset = shl i32 %tid, 4
+ %byte.offset64 = zext i32 %byte.offset to i64
+ %p.div = getelementptr i8, ptr addrspace(4) %p, i64 %byte.offset64
+ %v = load <4 x i32>, ptr addrspace(4) %p.div, align 8
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store volatile i32 %e1, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out0, i32 1
+ store volatile i32 %e2, ptr addrspace(1) %out1, align 4
+ ret void
+}
+
+define amdgpu_kernel void @keep_wide_v5i32_load(ptr addrspace(1) %out,
+ ptr addrspace(4) %p) {
+; ISEL-LABEL: name: keep_wide_v5i32_load
+; ISEL: GLOBAL_LOAD_DWORDX4{{.*}}load (s128)
+; ISEL-NOT: GLOBAL_LOAD_DWORDX3
+;
+; ISA-LABEL: keep_wide_v5i32_load:
+; ISA: global_load_b128
+; ISA-NOT: global_load_b96
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %byte.offset = shl i32 %tid, 5
+ %byte.offset64 = zext i32 %byte.offset to i64
+ %p.div = getelementptr i8, ptr addrspace(4) %p, i64 %byte.offset64
+ %v = load <5 x i32>, ptr addrspace(4) %p.div, align 8
+ %e0 = extractelement <5 x i32> %v, i32 0
+ %e1 = extractelement <5 x i32> %v, i32 1
+ %e2 = extractelement <5 x i32> %v, i32 2
+ %e3 = extractelement <5 x i32> %v, i32 3
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store volatile i32 %e0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out0, i32 1
+ store volatile i32 %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out0, i32 2
+ store volatile i32 %e2, ptr addrspace(1) %out2, align 4
+ %out3 = getelementptr i32, ptr addrspace(1) %out0, i32 3
+ store volatile i32 %e3, ptr addrspace(1) %out3, align 4
+ ret void
+}
+
+define amdgpu_kernel void @shrink_local_v4i32_load_to_v3i32(ptr addrspace(1) %out,
+ ptr addrspace(3) %p) {
+; ISEL-LABEL: name: shrink_local_v4i32_load_to_v3i32
+; ISEL: DS_READ_B96{{.*}}load (s96)
+; ISEL-NOT: DS_READ_B128
+;
+; ISA-LABEL: shrink_local_v4i32_load_to_v3i32:
+; ISA: ds_load_b96
+; ISA-NOT: ds_load_b128
+entry:
+ %v = load <4 x i32>, ptr addrspace(3) %p, align 16
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ store volatile i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store volatile i32 %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store volatile i32 %e2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define amdgpu_kernel void @shrink_flat_v4i32_load_to_v3i32(ptr addrspace(1) %out,
+ ptr %p) {
+; ISEL-LABEL: name: shrink_flat_v4i32_load_to_v3i32
+; ISEL: FLAT_LOAD_DWORDX3{{.*}}load (s96)
+; ISEL-NOT: FLAT_LOAD_DWORDX4
+;
+; ISA-LABEL: shrink_flat_v4i32_load_to_v3i32:
+; ISA: flat_load_b96
+; ISA-NOT: flat_load_b128
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %byte.offset = shl i32 %tid, 4
+ %byte.offset64 = zext i32 %byte.offset to i64
+ %p.div = getelementptr i8, ptr %p, i64 %byte.offset64
+ %v = load <4 x i32>, ptr %p.div, align 8
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store volatile i32 %e0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out0, i32 1
+ store volatile i32 %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out0, i32 2
+ store volatile i32 %e2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define amdgpu_kernel void @keep_underaligned_local_v4i32_load(ptr addrspace(1) %out,
+ ptr addrspace(3) %p) {
+; ISEL-LABEL: name: keep_underaligned_local_v4i32_load
+; ISEL-NOT: DS_READ_B96
+; ISEL: DS_READ
+;
+; ISA-LABEL: keep_underaligned_local_v4i32_load:
+; ISA-NOT: ds_load_b96
+; ISA: ds_load
+entry:
+ %v = load <4 x i32>, ptr addrspace(3) %p, align 4
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ store volatile i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store volatile i32 %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store volatile i32 %e2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define amdgpu_kernel void @shrink_private_v4i32_load_to_v3i32(ptr addrspace(1) %out,
+ ptr addrspace(5) %p) {
+; ISEL-LABEL: name: shrink_private_v4i32_load_to_v3i32
+; ISEL: SCRATCH_LOAD_DWORDX3{{.*}}load (s96)
+; ISEL-NOT: SCRATCH_LOAD_DWORDX4
+; PRIVATE8-LABEL: name: shrink_private_v4i32_load_to_v3i32
+; PRIVATE8-NOT: SCRATCH_LOAD_DWORDX3
+; PRIVATE8: BUFFER_LOAD_DWORDX2{{.*}}load (s64)
+; PRIVATE8: BUFFER_LOAD_DWORD{{.*}}load (s32)
+;
+; ISA-LABEL: shrink_private_v4i32_load_to_v3i32:
+; ISA: scratch_load_b96
+; ISA-NOT: scratch_load_b128
+entry:
+ %v = load <4 x i32>, ptr addrspace(5) %p, align 16
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ store volatile i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store volatile i32 %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store volatile i32 %e2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define amdgpu_kernel void @keep_smrd_constant_load(ptr addrspace(1) %out,
+ ptr addrspace(4) %p) {
+; ISEL-LABEL: name: keep_smrd_constant_load
+; ISEL-NOT: S_LOAD_DWORDX3
+; ISEL: S_LOAD_DWORDX4{{.*}}load (s128){{.*}}addrspace 4
+; ISEL-NOT: S_LOAD_DWORDX3
+; GFX11-SCALAR-LABEL: name: keep_smrd_constant_load
+; GFX11-SCALAR-NOT: S_LOAD_DWORDX3
+; GFX11-SCALAR-NOT: GLOBAL_LOAD_DWORDX3
+; GFX11-SCALAR: S_LOAD_DWORDX4{{.*}}load (s128){{.*}}addrspace 4
+; GFX11-SCALAR-NOT: S_LOAD_DWORDX3
+; GFX11-SCALAR-NOT: GLOBAL_LOAD_DWORDX3
+entry:
+ %v = load <2 x i64>, ptr addrspace(4) %p, align 16
+ %e0 = extractelement <2 x i64> %v, i32 0
+ %e1 = extractelement <2 x i64> %v, i32 1
+ %d0 = trunc i64 %e0 to i32
+ %e0hi = lshr i64 %e0, 32
+ %d1 = trunc i64 %e0hi to i32
+ %d2 = trunc i64 %e1 to i32
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 0
+ store volatile i32 %d0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store volatile i32 %d1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store volatile i32 %d2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define amdgpu_kernel void @keep_smrd_uniform_global_load(ptr addrspace(1) %out,
+ ptr addrspace(1) %p) {
+; ISEL-LABEL: name: keep_smrd_uniform_global_load
+; ISEL-NOT: S_LOAD_DWORDX3
+; ISEL: S_LOAD_DWORDX4{{.*}}load (s128) from %ir.2
+; ISEL-NOT: S_LOAD_DWORDX3
+; GFX11-SCALAR-LABEL: name: keep_smrd_uniform_global_load
+; GFX11-SCALAR-NOT: S_LOAD_DWORDX3
+; GFX11-SCALAR-NOT: GLOBAL_LOAD_DWORDX3
+; GFX11-SCALAR: S_LOAD_DWORDX4{{.*}}load (s128) from %ir.2
+; GFX11-SCALAR-NOT: S_LOAD_DWORDX3
+; GFX11-SCALAR-NOT: GLOBAL_LOAD_DWORDX3
+entry:
+ %v = load <2 x i64>, ptr addrspace(1) %p, align 16, !invariant.load !0
+ %e0 = extractelement <2 x i64> %v, i32 0
+ %e1 = extractelement <2 x i64> %v, i32 1
+ %d0 = trunc i64 %e0 to i32
+ %e0hi = lshr i64 %e0, 32
+ %d1 = trunc i64 %e0hi to i32
+ %d2 = trunc i64 %e1 to i32
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 0
+ store volatile i32 %d0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store volatile i32 %d1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store volatile i32 %d2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index c7e7927c37da3..fc829f0c4a6c4 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -466,7 +466,7 @@ define amdgpu_kernel void @ashr_i64_2(ptr addrspace(1) %out, ptr addrspace(1) %i
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s8, s2
; VI-NEXT: s_mov_b32 s9, s3
-; VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; VI-NEXT: buffer_load_dwordx3 v[0:2], off, s[8:11], 0
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/vector_range_metadata.ll b/llvm/test/CodeGen/AMDGPU/vector_range_metadata.ll
index 9053b8f9d4fe4..67d3272fc5da6 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_range_metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_range_metadata.ll
@@ -34,11 +34,8 @@ define <2 x i64> @test_add2x64(ptr %a_ptr, ptr %b_ptr) {
; CHECK-LABEL: test_add2x64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_load_dwordx4 v[6:9], v[2:3]
-; CHECK-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; CHECK-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; CHECK-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; CHECK-NEXT: flat_load_dwordx2 v[6:7], v[2:3]
; CHECK-NEXT: v_mov_b32_e32 v2, 48
; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -88,11 +85,8 @@ define <3 x i64> @test_add3x64(ptr %a_ptr, ptr %b_ptr) {
; CHECK-LABEL: test_add3x64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: flat_load_dwordx4 v[6:9], v[2:3]
-; CHECK-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; CHECK-NEXT: ; kill: killed $vgpr2 killed $vgpr3
+; CHECK-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; CHECK-NEXT: flat_load_dwordx2 v[6:7], v[2:3]
; CHECK-NEXT: v_mov_b32_e32 v2, 48
; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index 6bf6d540299f1..a2161dc40126a 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -1940,28 +1940,23 @@ define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1)
; GX900-LABEL: shuffle_v6f16_452367:
; GX900: ; %bb.0:
; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GX900-NEXT: v_mov_b32_e32 v6, v1
-; GX900-NEXT: v_mov_b32_e32 v5, v0
-; GX900-NEXT: v_mov_b32_e32 v4, v3
-; GX900-NEXT: v_mov_b32_e32 v3, v2
-; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
-; GX900-NEXT: global_load_dword v7, v[3:4], off
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v6, v[2:3], off
; GX900-NEXT: s_waitcnt vmcnt(1)
-; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: v_mov_b32_e32 v0, v5
+; GX900-NEXT: v_mov_b32_e32 v1, v4
; GX900-NEXT: s_waitcnt vmcnt(0)
-; GX900-NEXT: v_mov_b32_e32 v2, v7
+; GX900-NEXT: v_mov_b32_e32 v2, v6
; GX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: shuffle_v6f16_452367:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx3 v[4:6], v[0:1], off
+; GFX942-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:4
; GFX942-NEXT: global_load_dword v4, v[2:3], off
-; GFX942-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX942-NEXT: ; kill: killed $vgpr2 killed $vgpr3
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_mov_b32_e32 v0, v6
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1969,28 +1964,23 @@ define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1)
; GFX10-LABEL: shuffle_v6f16_452367:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v6, v1
-; GFX10-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
-; GFX10-NEXT: global_load_dword v7, v[3:4], off
+; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:4
+; GFX10-NEXT: global_load_dword v6, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v7
+; GFX10-NEXT: v_mov_b32_e32 v2, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v6f16_452367:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX11-NEXT: global_load_b96 v[0:2], v[0:1], off
-; GFX11-NEXT: global_load_b32 v3, v[3:4], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:4
+; GFX11-NEXT: global_load_b32 v2, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v4
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <6 x half>, ptr addrspace(1) %arg0
%val1 = load <6 x half>, ptr addrspace(1) %arg1
@@ -5134,28 +5124,23 @@ define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace
; GX900-LABEL: shuffle_v6bf16_452367:
; GX900: ; %bb.0:
; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GX900-NEXT: v_mov_b32_e32 v6, v1
-; GX900-NEXT: v_mov_b32_e32 v5, v0
-; GX900-NEXT: v_mov_b32_e32 v4, v3
-; GX900-NEXT: v_mov_b32_e32 v3, v2
-; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
-; GX900-NEXT: global_load_dword v7, v[3:4], off
+; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:4
+; GX900-NEXT: global_load_dword v6, v[2:3], off
; GX900-NEXT: s_waitcnt vmcnt(1)
-; GX900-NEXT: v_mov_b32_e32 v0, v2
+; GX900-NEXT: v_mov_b32_e32 v0, v5
+; GX900-NEXT: v_mov_b32_e32 v1, v4
; GX900-NEXT: s_waitcnt vmcnt(0)
-; GX900-NEXT: v_mov_b32_e32 v2, v7
+; GX900-NEXT: v_mov_b32_e32 v2, v6
; GX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: shuffle_v6bf16_452367:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx3 v[4:6], v[0:1], off
+; GFX942-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:4
; GFX942-NEXT: global_load_dword v4, v[2:3], off
-; GFX942-NEXT: ; kill: killed $vgpr0 killed $vgpr1
-; GFX942-NEXT: ; kill: killed $vgpr2 killed $vgpr3
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_mov_b32_e32 v0, v6
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: v_mov_b32_e32 v0, v7
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -5163,28 +5148,23 @@ define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace
; GFX10-LABEL: shuffle_v6bf16_452367:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v6, v1
-; GFX10-NEXT: v_mov_b32_e32 v5, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: global_load_dwordx3 v[0:2], v[5:6], off
-; GFX10-NEXT: global_load_dword v7, v[3:4], off
+; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:4
+; GFX10-NEXT: global_load_dword v6, v[2:3], off
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v7
+; GFX10-NEXT: v_mov_b32_e32 v2, v6
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: shuffle_v6bf16_452367:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX11-NEXT: global_load_b96 v[0:2], v[0:1], off
-; GFX11-NEXT: global_load_b32 v3, v[3:4], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:4
+; GFX11-NEXT: global_load_b32 v2, v[2:3], off
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v4
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <6 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <6 x bfloat>, ptr addrspace(1) %arg1
More information about the llvm-commits
mailing list