[llvm] AMDGPU: Shrink extracted dword vector loads (PR #203484)

via llvm-commits llvm-commits at lists.llvm.org
Fri Jun 12 02:28:46 PDT 2026


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-backend-amdgpu

Author: tianhbai

<details>
<summary>Changes</summary>

## Summary

Teach AMDGPU SelectionDAG to shrink simple dword vector loads that feed constant-index `extract_vector_elt` users.

When the used lanes form one contiguous run, the combine replaces the original load with one narrower `2`-`4` dword load at the adjusted byte offset and rewrites the extract users.

The transform stays conservative:

- only handles simple normal `i32` vector loads
- only shrinks one contiguous used-lane range
- only emits 2-4 dword loads
- checks `dwordx3`, local, and private load support
- skips loads that may select to SMRD, avoiding cases where this could block later `s_load_dwordx8` / `s_load_dwordx16` merging

This enables VMEM/flat/local/private cases such as `global_load_b128` to become `global_load_b96` when the unused dword is proven unnecessary.

## Tests

- Built `llc`
- `llvm-lit -sv llvm/test/CodeGen/AMDGPU`

## AI Tool Usage

AI tools were used to assist with code exploration, drafting, and review of this PR. I reviewed and validated the changes.


---

Patch is 75.12 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/203484.diff


14 Files Affected:

- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+169-1) 
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+10-10) 
- (modified) llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll (+2-2) 
- (modified) llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll (+72-58) 
- (modified) llvm/test/CodeGen/AMDGPU/function-returns.ll (+2-2) 
- (modified) llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll (+23-21) 
- (modified) llvm/test/CodeGen/AMDGPU/idot4u.ll (+72-82) 
- (modified) llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll (+24-26) 
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll (+4-4) 
- (modified) llvm/test/CodeGen/AMDGPU/permute_i8.ll (+8-8) 
- (added) llvm/test/CodeGen/AMDGPU/shrink-extracted-dword-load.ll (+357) 
- (modified) llvm/test/CodeGen/AMDGPU/sra.ll (+1-1) 
- (modified) llvm/test/CodeGen/AMDGPU/vector_range_metadata.ll (+4-10) 
- (modified) llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll (+32-52) 


``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a6dc58e6da263..4562e329b6a6a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16505,6 +16505,171 @@ bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
       EltSize, NumElem, Idx->isDivergent(), getSubtarget());
 }
 
+// Return true if this load may select to an SMRD instruction.
+static bool maySelectSMRD(const LoadSDNode *Ld, const GCNSubtarget *Subtarget,
+                          const SITargetLowering *TLI) {
+  const MachineMemOperand *MMO = Ld->getMemOperand();
+  if (!MMO->getSize().hasValue())
+    return false;
+
+  Align MinScalarAlign(
+      std::min(MMO->getSize().getValue().getKnownMinValue(), uint64_t(4)));
+  if (Ld->getAlign() < MinScalarAlign)
+    return false;
+
+  if (Ld->isDivergent() && !AMDGPU::isUniformMMO(MMO))
+    return false;
+
+  switch (Ld->getAddressSpace()) {
+  case AMDGPUAS::CONSTANT_ADDRESS:
+  case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
+    return true;
+  case AMDGPUAS::GLOBAL_ADDRESS:
+    return Ld->getMemOperand()->isInvariant() ||
+           (Subtarget->getScalarizeGlobalBehavior() &&
+            TLI->isMemOpHasNoClobberedMemOperand(Ld));
+  default:
+    return false;
+  }
+}
+
+static bool canShrinkDwordVectorLoad(const LoadSDNode *Ld,
+                                     const GCNSubtarget *Subtarget,
+                                     const SITargetLowering *TLI,
+                                     unsigned NewNumDwords,
+                                     uint64_t ByteOffset) {
+  // Keep this as one vector load: no scalar load and no wider-than-128-bit
+  // load.
+  if (NewNumDwords < 2 || NewNumDwords > 4)
+    return false;
+
+  // Avoid blocking later SMRD merging into wider x8/x16 loads.
+  if (maySelectSMRD(Ld, Subtarget, TLI))
+    return false;
+
+  if (NewNumDwords == 3 && !Subtarget->hasDwordx3LoadStores())
+    return false;
+
+  unsigned AS = Ld->getAddressSpace();
+  Align NewAlign = commonAlignment(Ld->getAlign(), ByteOffset);
+  switch (AS) {
+  case AMDGPUAS::CONSTANT_ADDRESS:
+  case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
+  case AMDGPUAS::GLOBAL_ADDRESS:
+  case AMDGPUAS::FLAT_ADDRESS:
+    return true;
+  case AMDGPUAS::LOCAL_ADDRESS: {
+    unsigned Fast = 0;
+    return TLI->allowsMisalignedMemoryAccessesImpl(
+               NewNumDwords * 32, AS, NewAlign,
+               Ld->getMemOperand()->getFlags(), &Fast) &&
+           Fast > 1;
+  }
+  case AMDGPUAS::PRIVATE_ADDRESS:
+    if (Subtarget->getMaxPrivateElementSize() < NewNumDwords * 4)
+      return false;
+
+    return TLI->allowsMisalignedMemoryAccessesImpl(
+        NewNumDwords * 32, AS, NewAlign,
+        Ld->getMemOperand()->getFlags());
+  default:
+    return false;
+  }
+}
+
+// Shrink a dword vector load when the used lanes form a contiguous range.
+static SDValue shrinkExtractedDwordLoad(SDNode *N,
+                                        TargetLowering::DAGCombinerInfo &DCI,
+                                        const GCNSubtarget *Subtarget,
+                                        const SITargetLowering *TLI) {
+  if (!DCI.isAfterLegalizeDAG())
+    return SDValue();
+
+  SDValue Vec = N->getOperand(0);
+  auto *Ld = dyn_cast<LoadSDNode>(Vec);
+  MVT OldVT = Vec.getSimpleValueType();
+  if (!Ld || !OldVT.isVector() || OldVT.getVectorElementType() != MVT::i32 ||
+      !Ld->isSimple() || !ISD::isNormalLoad(Ld))
+    return SDValue();
+
+  unsigned OldNumLanes = OldVT.getVectorNumElements();
+  APInt UsedLanes = APInt::getZero(OldNumLanes);
+  SmallVector<SDNode *, 4> Extracts;
+  for (SDUse &Use : Ld->uses()) {
+    if (Use.getResNo() != 0)
+      continue;
+
+    SDNode *User = Use.getUser();
+    if (User->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+      return SDValue();
+
+    auto *Idx = dyn_cast<ConstantSDNode>(User->getOperand(1));
+    if (!Idx)
+      return SDValue();
+
+    uint64_t Lane = Idx->getZExtValue();
+    if (Lane >= OldNumLanes)
+      return SDValue();
+
+    UsedLanes.setBit(Lane);
+    Extracts.push_back(User);
+  }
+
+  if (Extracts.empty())
+    return SDValue();
+
+  // The used lane mask must be one contiguous run. This also returns the start
+  // lane and run length for the replacement load.
+  unsigned FirstLane = 0;
+  unsigned NewNumLanes = 0;
+  if (!UsedLanes.isShiftedMask(FirstLane, NewNumLanes))
+    return SDValue();
+
+  if (NewNumLanes >= OldNumLanes)
+    return SDValue();
+
+  uint64_t ByteOffset = FirstLane * 4;
+  if (!canShrinkDwordVectorLoad(Ld, Subtarget, TLI, NewNumLanes, ByteOffset))
+    return SDValue();
+
+  SDLoc SL(Ld);
+  SelectionDAG &DAG = DCI.DAG;
+  MachineFunction &MF = DAG.getMachineFunction();
+  MVT NewVT = MVT::getVectorVT(MVT::i32, NewNumLanes);
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      Ld->getMemOperand(), ByteOffset, NewVT.getStoreSize());
+  SDValue NewBase =
+      ByteOffset == 0
+          ? Ld->getBasePtr()
+          : DAG.getMemBasePlusOffset(Ld->getBasePtr(),
+                                     TypeSize::getFixed(ByteOffset), SL);
+  SDValue NewLoad = DAG.getLoad(NewVT, SL, Ld->getChain(), NewBase, MMO);
+  DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
+  DCI.AddToWorklist(NewLoad.getNode());
+
+  for (SDNode *Extract : Extracts) {
+    if (Extract == N)
+      continue;
+
+    unsigned Lane =
+        cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue();
+    SDValue NewIdx =
+        DAG.getVectorIdxConstant(Lane - FirstLane, SDLoc(Extract));
+    SDNode *NewExtract = DAG.UpdateNodeOperands(Extract, NewLoad, NewIdx);
+    DCI.AddToWorklist(NewExtract);
+    if (NewExtract != Extract) {
+      DAG.ReplaceAllUsesWith(SDValue(Extract, 0), SDValue(NewExtract, 0));
+      DAG.RemoveDeadNode(Extract);
+    }
+  }
+
+  auto *Idx = cast<ConstantSDNode>(N->getOperand(1));
+  SDValue NewIdx =
+      DAG.getVectorIdxConstant(Idx->getZExtValue() - FirstLane, SDLoc(N));
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
+                     NewLoad, NewIdx);
+}
+
 SDValue
 SITargetLowering::performExtractVectorEltCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
@@ -16628,8 +16793,11 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
     }
   }
 
-  if (!DCI.isBeforeLegalize())
+  if (!DCI.isBeforeLegalize()) {
+    if (SDValue Shrunk = shrinkExtractedDwordLoad(N, DCI, Subtarget, this))
+      return Shrunk;
     return SDValue();
+  }
 
   // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
   // elements. This exposes more load reduction opportunities by replacing
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 2ac5574be0def..8b8e7ed21815e 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -5897,7 +5897,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
 ; GFX8-LABEL: global_extload_v5bf16_to_v5f32:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_load_dwordx4 v[2:5], v[0:1]
+; GFX8-NEXT:    flat_load_dwordx3 v[2:4], v[0:1]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
@@ -5909,7 +5909,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
 ; GFX9-LABEL: global_extload_v5bf16_to_v5f32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx3 v[2:4], v[0:1], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
@@ -5921,7 +5921,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
 ; GFX10-LABEL: global_extload_v5bf16_to_v5f32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off
+; GFX10-NEXT:    global_load_dwordx3 v[2:4], v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
@@ -5933,7 +5933,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
 ; GFX11-LABEL: global_extload_v5bf16_to_v5f32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    global_load_b128 v[2:5], v[0:1], off
+; GFX11-NEXT:    global_load_b96 v[2:4], v[0:1], off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
@@ -5946,7 +5946,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b128 v[2:5], v[0:1], off
+; GFX1250-NEXT:    global_load_b96 v[2:4], v[0:1], off
 ; GFX1250-NEXT:    s_wait_loadcnt 0x0
 ; GFX1250-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
@@ -7125,7 +7125,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
 ; GFX8-LABEL: global_extload_v5bf16_to_v5f64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT:    flat_load_dwordx3 v[0:2], v[0:1]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
@@ -7142,7 +7142,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
 ; GFX9-LABEL: global_extload_v5bf16_to_v5f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT:    global_load_dwordx3 v[0:2], v[0:1], off
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 16, v0
 ; GFX9-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
@@ -7159,7 +7159,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
 ; GFX10-LABEL: global_extload_v5bf16_to_v5f64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off
+; GFX10-NEXT:    global_load_dwordx3 v[2:4], v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
@@ -7176,7 +7176,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
 ; GFX11-LABEL: global_extload_v5bf16_to_v5f64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    global_load_b128 v[2:5], v[0:1], off
+; GFX11-NEXT:    global_load_b96 v[2:4], v[0:1], off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX11-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
@@ -7194,7 +7194,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b128 v[2:5], v[0:1], off
+; GFX1250-NEXT:    global_load_b96 v[2:4], v[0:1], off
 ; GFX1250-NEXT:    s_wait_loadcnt 0x0
 ; GFX1250-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-NEXT:    v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_lshlrev_b32 v5, 16, v3
diff --git a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
index 3126491559d0a..a7e5949e5501a 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
@@ -4,8 +4,8 @@
 ; This caused failure in infinite cycle in Selection DAG (combine) due to missing insert_subvector.
 ;
 ; CHECK-LABEL: name: test1
-; CHECK: GLOBAL_LOAD_DWORDX4
-; CHECK: GLOBAL_LOAD_DWORDX4
+; CHECK: GLOBAL_LOAD_DWORDX2
+; CHECK: GLOBAL_LOAD_DWORDX2
 ; CHECK: GLOBAL_STORE_DWORDX4
 define protected amdgpu_kernel void @test1(ptr addrspace(4) %ptr) local_unnamed_addr !kernel_arg_addr_space !0 !kernel_arg_access_qual !1 !kernel_arg_type !2 !kernel_arg_base_type !2 !kernel_arg_type_qual !3 !kernel_arg_name !4 {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 57fe32256e624..a0792a3252cfa 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -506,7 +506,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
 ; GFX90A-SDAG-MUBUF-NEXT:    flat_load_dwordx4 v[32:35], v[6:7] offset:16
 ; GFX90A-SDAG-MUBUF-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
 ; GFX90A-SDAG-MUBUF-NEXT:    flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-SDAG-MUBUF-NEXT:    flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-MUBUF-NEXT:    flat_load_dwordx3 v[48:50], v[4:5]
 ; GFX90A-SDAG-MUBUF-NEXT:    v_add_co_u32_e32 v0, vcc, 48, v2
 ; GFX90A-SDAG-MUBUF-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
 ; GFX90A-SDAG-MUBUF-NEXT:    v_add_co_u32_e32 v4, vcc, 0x88, v2
@@ -544,7 +544,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
 ; GFX90A-SDAG-FLATSCR-NEXT:    flat_load_dwordx4 v[32:35], v[6:7] offset:16
 ; GFX90A-SDAG-FLATSCR-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
 ; GFX90A-SDAG-FLATSCR-NEXT:    flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-SDAG-FLATSCR-NEXT:    flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-FLATSCR-NEXT:    flat_load_dwordx3 v[48:50], v[4:5]
 ; GFX90A-SDAG-FLATSCR-NEXT:    v_add_co_u32_e32 v0, vcc, 48, v2
 ; GFX90A-SDAG-FLATSCR-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
 ; GFX90A-SDAG-FLATSCR-NEXT:    v_add_co_u32_e32 v4, vcc, 0x88, v2
@@ -570,30 +570,29 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
 ; GFX10-SDAG-NEXT:    v_lshlrev_b64 v[4:5], 2, v[4:5]
 ; GFX10-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX10-SDAG-NEXT:    v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT:    v_add_co_u32 v32, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e64 v33, null, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT:    v_add_co_u32 v36, vcc_lo, 0x8c, v0
 ; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
 ; GFX10-SDAG-NEXT:    s_clause 0x8
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[4:7], v[36:37] offset:80
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[8:11], v[36:37] offset:96
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[12:15], v[36:37] offset:48
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[16:19], v[36:37] offset:64
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[20:23], v[36:37] offset:16
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[24:27], v[36:37] offset:32
+; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[4:7], v[32:33] offset:48
+; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[8:11], v[32:33] offset:96
+; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[12:15], v[32:33] offset:64
+; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[16:19], v[32:33] offset:80
+; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[20:23], v[32:33] offset:16
+; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[24:27], v[32:33] offset:32
 ; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[28:31], v[0:1] offset:12
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[32:35], v[36:37]
-; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[36:39], v[36:37] offset:112
+; GFX10-SDAG-NEXT:    flat_load_dwordx4 v[32:35], v[32:33]
+; GFX10-SDAG-NEXT:    flat_load_dwordx3 v[36:38], v[36:37]
 ; GFX10-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v2, 48
 ; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
 ; GFX10-SDAG-NEXT:    v_add_co_u32 v48, vcc_lo, 0x88, v2
 ; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(8) lgkmcnt(8)
-; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[4:7] offset:48
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(7) lgkmcnt(8)
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(5)
+; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[16:19] offset:48
 ; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[8:11] offset:64
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(8)
-; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(8)
-; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[2:3], v[4:7] offset:64
+; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[0:1], v[12:15] offset:32
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(4) lgkmcnt(8)
 ; GFX10-SDAG-NEXT:    flat_store_dwordx4 v[2:3], v[20:23] offset:32
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(3) lgkmcnt(8)
@@ -638,7 +637,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
 ; GFX942-SDAG-NEXT:    flat_load_dwordx4 v[52:55], v[0:1] offset:12
 ; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0x8c
 ; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942-SDAG-NEXT:    flat_load_dwordx4 a[0:3], v[0:1]
+; GFX942-SDAG-NEXT:    flat_load_dwordx3 a[0:2], v[0:1]
 ; GFX942-SDAG-NEXT:    s_mov_b64 s[0:1], 0x60
 ; GFX942-SDAG-NEXT:    s_mov_b64 s[2:3], 0x70
 ; GFX942-SDAG-NEXT:    s_mov_b64 s[4:5], 0x50
@@ -671,38 +670,45 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
 ; GFX11-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX11-SDAG-NEXT:    v_add_co_u32 v36, vcc_lo, v0, 28
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX11-SDAG-NEXT:    s_clause 0x7
-; GFX11-SDAG-NEXT:    flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX11-SDAG-NEXT:    flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX11-SDAG-NEXT:    flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX11-SDAG-NEXT:    flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX11-SDAG-NEXT:    flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX11-SDAG-NEXT:    flat_load_b128 v[24:27], v[36:37]
+; GFX11-SDAG-NEXT:    v_add_co_u32 v32, vcc_lo, v0, 28
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_add_co_ci_u32_e64 v33, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT:    v_add_co_u32 v34, vcc_lo, 0x8c, v0
+; GFX11-SDAG-NEXT:    v_add_co_ci_u32_e64 v35, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT:    s_clause 0x8
+; GFX11-SDAG-NEXT:    flat_load_b128 v[4:7], v[32:33] offset:96
+; GFX11-SDAG-NEXT:    flat_load_b128 v[8:11], v[32:33] offset:64
+; GFX11-SDAG-NEXT:    flat_load_b128 v[12:15], v[32:33] offset:80
+; GFX11-SDAG-NEXT:    flat_load_b128 v[16:19], v[32:33] offset:32
+; GFX11-SDAG-NEXT:    flat_load_b128 v[20:23], v[32:33] offset:16
+; GFX11-SDAG-NEXT:    flat_load_b128 v[24:27], v[32:33]
 ; GFX11-SDAG-NEXT:    flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX11-SDAG-NEXT:    flat_load_b128 v[32:35], v[36:37] offset:112
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    flat_load_b128 v[35:38], v[36:37] offset:48
+; GFX11-SDAG-NEXT:    flat_load_b96 v[36:38], v[34:35]
+; GFX11-SDAG-NEXT:    flat_load_b128 v[32:35], v[32:33] offset:48
 ; GFX11-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v2, 48
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
 ; GFX11-SDAG-NEXT:    v_add_co_u32 v48, vcc_lo, 0x88, v2
 ; GFX11-SDAG-NEXT:    v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX11-SDAG-NEXT:    s_clause 0x7
-; GFX11-SDAG-NEXT:    flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX11-SDAG-NEXT:    flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX11-SDAG-NEXT:    flat_store_b128 v[0:1], v[12:15] offset:32
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(6) lgkmcnt(6)
+; GFX11-SDAG-NEXT:    s_clause 0x2
+; GFX11-SDAG-NEXT:    flat_store_b128 v[0:1], v[12:15] offset:48
+; GFX11-SDAG-NEXT:    flat_store_b128 v[0:1], v[4:7] offset:64
+; GFX11-SDAG-NEXT:    flat_store_b128 v[0:1], v[8:11] offset:32
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(5) lgkmcnt(8)
 ; GFX11-SDAG-NEXT:    flat_store_b128 v[0:1], v[16:19]
+; GFX11-SD...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/203484


More information about the llvm-commits mailing list