[llvm] AMDGPU: Shrink extracted dword vector loads (PR #203484)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 02:28:46 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: tianhbai
<details>
<summary>Changes</summary>
## Summary
Teach AMDGPU SelectionDAG to shrink simple dword vector loads that feed constant-index `extract_vector_elt` users.
When the used lanes form one contiguous run, the combine replaces the original load with one narrower `2`-`4` dword load at the adjusted byte offset and rewrites the extract users.
The transform stays conservative:
- only handles simple normal `i32` vector loads
- only shrinks one contiguous used-lane range
- only emits 2-4 dword loads
- checks `dwordx3`, local, and private load support
- skips loads that may select to SMRD, avoiding cases where this could block later `s_load_dwordx8` / `s_load_dwordx16` merging
This enables VMEM/flat/local/private cases such as `global_load_b128` to become `global_load_b96` when the unused dword is proven unnecessary.
## Tests
- Built `llc`
- `llvm-lit -sv llvm/test/CodeGen/AMDGPU`
## AI Tool Usage
AI tools were used to assist with code exploration, drafting, and review of this PR. I reviewed and validated the changes.
---
Patch is 75.12 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/203484.diff
14 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIISelLowering.cpp (+169-1)
- (modified) llvm/test/CodeGen/AMDGPU/bf16.ll (+10-10)
- (modified) llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll (+2-2)
- (modified) llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll (+72-58)
- (modified) llvm/test/CodeGen/AMDGPU/function-returns.ll (+2-2)
- (modified) llvm/test/CodeGen/AMDGPU/gfx-callable-return-types.ll (+23-21)
- (modified) llvm/test/CodeGen/AMDGPU/idot4u.ll (+72-82)
- (modified) llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll (+24-26)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll (+4-4)
- (modified) llvm/test/CodeGen/AMDGPU/permute_i8.ll (+8-8)
- (added) llvm/test/CodeGen/AMDGPU/shrink-extracted-dword-load.ll (+357)
- (modified) llvm/test/CodeGen/AMDGPU/sra.ll (+1-1)
- (modified) llvm/test/CodeGen/AMDGPU/vector_range_metadata.ll (+4-10)
- (modified) llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll (+32-52)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a6dc58e6da263..4562e329b6a6a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16505,6 +16505,171 @@ bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
EltSize, NumElem, Idx->isDivergent(), getSubtarget());
}
+// Return true if this load may select to an SMRD instruction.
+static bool maySelectSMRD(const LoadSDNode *Ld, const GCNSubtarget *Subtarget,
+ const SITargetLowering *TLI) {
+ const MachineMemOperand *MMO = Ld->getMemOperand();
+ if (!MMO->getSize().hasValue())
+ return false;
+
+ Align MinScalarAlign(
+ std::min(MMO->getSize().getValue().getKnownMinValue(), uint64_t(4)));
+ if (Ld->getAlign() < MinScalarAlign)
+ return false;
+
+ if (Ld->isDivergent() && !AMDGPU::isUniformMMO(MMO))
+ return false;
+
+ switch (Ld->getAddressSpace()) {
+ case AMDGPUAS::CONSTANT_ADDRESS:
+ case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
+ return true;
+ case AMDGPUAS::GLOBAL_ADDRESS:
+ return Ld->getMemOperand()->isInvariant() ||
+ (Subtarget->getScalarizeGlobalBehavior() &&
+ TLI->isMemOpHasNoClobberedMemOperand(Ld));
+ default:
+ return false;
+ }
+}
+
+static bool canShrinkDwordVectorLoad(const LoadSDNode *Ld,
+ const GCNSubtarget *Subtarget,
+ const SITargetLowering *TLI,
+ unsigned NewNumDwords,
+ uint64_t ByteOffset) {
+ // Keep this as one vector load: no scalar load and no wider-than-128-bit
+ // load.
+ if (NewNumDwords < 2 || NewNumDwords > 4)
+ return false;
+
+ // Avoid blocking later SMRD merging into wider x8/x16 loads.
+ if (maySelectSMRD(Ld, Subtarget, TLI))
+ return false;
+
+ if (NewNumDwords == 3 && !Subtarget->hasDwordx3LoadStores())
+ return false;
+
+ unsigned AS = Ld->getAddressSpace();
+ Align NewAlign = commonAlignment(Ld->getAlign(), ByteOffset);
+ switch (AS) {
+ case AMDGPUAS::CONSTANT_ADDRESS:
+ case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
+ case AMDGPUAS::GLOBAL_ADDRESS:
+ case AMDGPUAS::FLAT_ADDRESS:
+ return true;
+ case AMDGPUAS::LOCAL_ADDRESS: {
+ unsigned Fast = 0;
+ return TLI->allowsMisalignedMemoryAccessesImpl(
+ NewNumDwords * 32, AS, NewAlign,
+ Ld->getMemOperand()->getFlags(), &Fast) &&
+ Fast > 1;
+ }
+ case AMDGPUAS::PRIVATE_ADDRESS:
+ if (Subtarget->getMaxPrivateElementSize() < NewNumDwords * 4)
+ return false;
+
+ return TLI->allowsMisalignedMemoryAccessesImpl(
+ NewNumDwords * 32, AS, NewAlign,
+ Ld->getMemOperand()->getFlags());
+ default:
+ return false;
+ }
+}
+
+// Shrink a dword vector load when the used lanes form a contiguous range.
+static SDValue shrinkExtractedDwordLoad(SDNode *N,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const GCNSubtarget *Subtarget,
+ const SITargetLowering *TLI) {
+ if (!DCI.isAfterLegalizeDAG())
+ return SDValue();
+
+ SDValue Vec = N->getOperand(0);
+ auto *Ld = dyn_cast<LoadSDNode>(Vec);
+ MVT OldVT = Vec.getSimpleValueType();
+ if (!Ld || !OldVT.isVector() || OldVT.getVectorElementType() != MVT::i32 ||
+ !Ld->isSimple() || !ISD::isNormalLoad(Ld))
+ return SDValue();
+
+ unsigned OldNumLanes = OldVT.getVectorNumElements();
+ APInt UsedLanes = APInt::getZero(OldNumLanes);
+ SmallVector<SDNode *, 4> Extracts;
+ for (SDUse &Use : Ld->uses()) {
+ if (Use.getResNo() != 0)
+ continue;
+
+ SDNode *User = Use.getUser();
+ if (User->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+ return SDValue();
+
+ auto *Idx = dyn_cast<ConstantSDNode>(User->getOperand(1));
+ if (!Idx)
+ return SDValue();
+
+ uint64_t Lane = Idx->getZExtValue();
+ if (Lane >= OldNumLanes)
+ return SDValue();
+
+ UsedLanes.setBit(Lane);
+ Extracts.push_back(User);
+ }
+
+ if (Extracts.empty())
+ return SDValue();
+
+ // The used lane mask must be one contiguous run. This also returns the start
+ // lane and run length for the replacement load.
+ unsigned FirstLane = 0;
+ unsigned NewNumLanes = 0;
+ if (!UsedLanes.isShiftedMask(FirstLane, NewNumLanes))
+ return SDValue();
+
+ if (NewNumLanes >= OldNumLanes)
+ return SDValue();
+
+ uint64_t ByteOffset = FirstLane * 4;
+ if (!canShrinkDwordVectorLoad(Ld, Subtarget, TLI, NewNumLanes, ByteOffset))
+ return SDValue();
+
+ SDLoc SL(Ld);
+ SelectionDAG &DAG = DCI.DAG;
+ MachineFunction &MF = DAG.getMachineFunction();
+ MVT NewVT = MVT::getVectorVT(MVT::i32, NewNumLanes);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ Ld->getMemOperand(), ByteOffset, NewVT.getStoreSize());
+ SDValue NewBase =
+ ByteOffset == 0
+ ? Ld->getBasePtr()
+ : DAG.getMemBasePlusOffset(Ld->getBasePtr(),
+ TypeSize::getFixed(ByteOffset), SL);
+ SDValue NewLoad = DAG.getLoad(NewVT, SL, Ld->getChain(), NewBase, MMO);
+ DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
+ DCI.AddToWorklist(NewLoad.getNode());
+
+ for (SDNode *Extract : Extracts) {
+ if (Extract == N)
+ continue;
+
+ unsigned Lane =
+ cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue();
+ SDValue NewIdx =
+ DAG.getVectorIdxConstant(Lane - FirstLane, SDLoc(Extract));
+ SDNode *NewExtract = DAG.UpdateNodeOperands(Extract, NewLoad, NewIdx);
+ DCI.AddToWorklist(NewExtract);
+ if (NewExtract != Extract) {
+ DAG.ReplaceAllUsesWith(SDValue(Extract, 0), SDValue(NewExtract, 0));
+ DAG.RemoveDeadNode(Extract);
+ }
+ }
+
+ auto *Idx = cast<ConstantSDNode>(N->getOperand(1));
+ SDValue NewIdx =
+ DAG.getVectorIdxConstant(Idx->getZExtValue() - FirstLane, SDLoc(N));
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
+ NewLoad, NewIdx);
+}
+
SDValue
SITargetLowering::performExtractVectorEltCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -16628,8 +16793,11 @@ SITargetLowering::performExtractVectorEltCombine(SDNode *N,
}
}
- if (!DCI.isBeforeLegalize())
+ if (!DCI.isBeforeLegalize()) {
+ if (SDValue Shrunk = shrinkExtractedDwordLoad(N, DCI, Subtarget, this))
+ return Shrunk;
return SDValue();
+ }
// Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
// elements. This exposes more load reduction opportunities by replacing
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 2ac5574be0def..8b8e7ed21815e 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -5897,7 +5897,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX8-LABEL: global_extload_v5bf16_to_v5f32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx4 v[2:5], v[0:1]
+; GFX8-NEXT: flat_load_dwordx3 v[2:4], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5909,7 +5909,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX9-LABEL: global_extload_v5bf16_to_v5f32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
+; GFX9-NEXT: global_load_dwordx3 v[2:4], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5921,7 +5921,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX10-LABEL: global_extload_v5bf16_to_v5f32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
+; GFX10-NEXT: global_load_dwordx3 v[2:4], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5933,7 +5933,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX11-LABEL: global_extload_v5bf16_to_v5f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX11-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX11-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
@@ -5946,7 +5946,7 @@ define <5 x float> @global_extload_v5bf16_to_v5f32(ptr addrspace(1) %ptr) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX1250-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2
@@ -7125,7 +7125,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX8-LABEL: global_extload_v5bf16_to_v5f64:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT: flat_load_dwordx3 v[0:2], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
@@ -7142,7 +7142,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX9-LABEL: global_extload_v5bf16_to_v5f64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
+; GFX9-NEXT: global_load_dwordx3 v[0:2], v[0:1], off
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
@@ -7159,7 +7159,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX10-LABEL: global_extload_v5bf16_to_v5f64:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
+; GFX10-NEXT: global_load_dwordx3 v[2:4], v[0:1], off
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -7176,7 +7176,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX11-LABEL: global_extload_v5bf16_to_v5f64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX11-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
@@ -7194,7 +7194,7 @@ define <5 x double> @global_extload_v5bf16_to_v5f64(ptr addrspace(1) %ptr) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b128 v[2:5], v[0:1], off
+; GFX1250-NEXT: global_load_b96 v[2:4], v[0:1], off
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v2 :: v_dual_lshlrev_b32 v5, 16, v3
diff --git a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
index 3126491559d0a..a7e5949e5501a 100644
--- a/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/bug-v4f64-subvector.ll
@@ -4,8 +4,8 @@
; This caused failure in infinite cycle in Selection DAG (combine) due to missing insert_subvector.
;
; CHECK-LABEL: name: test1
-; CHECK: GLOBAL_LOAD_DWORDX4
-; CHECK: GLOBAL_LOAD_DWORDX4
+; CHECK: GLOBAL_LOAD_DWORDX2
+; CHECK: GLOBAL_LOAD_DWORDX2
; CHECK: GLOBAL_STORE_DWORDX4
define protected amdgpu_kernel void @test1(ptr addrspace(4) %ptr) local_unnamed_addr !kernel_arg_addr_space !0 !kernel_arg_access_qual !1 !kernel_arg_type !2 !kernel_arg_base_type !2 !kernel_arg_type_qual !3 !kernel_arg_name !4 {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 57fe32256e624..a0792a3252cfa 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -506,7 +506,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-MUBUF-NEXT: flat_load_dwordx3 v[48:50], v[4:5]
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
; GFX90A-SDAG-MUBUF-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX90A-SDAG-MUBUF-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
@@ -544,7 +544,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[32:35], v[6:7] offset:16
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[36:39], v[0:1] offset:12
-; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx4 v[48:51], v[4:5]
+; GFX90A-SDAG-FLATSCR-NEXT: flat_load_dwordx3 v[48:50], v[4:5]
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v0, vcc, 48, v2
; GFX90A-SDAG-FLATSCR-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX90A-SDAG-FLATSCR-NEXT: v_add_co_u32_e32 v4, vcc, 0x88, v2
@@ -570,30 +570,29 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX10-SDAG-NEXT: v_lshlrev_b64 v[4:5], 2, v[4:5]
; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX10-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT: v_add_co_u32 v32, vcc_lo, v0, 28
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, 0x8c, v0
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
; GFX10-SDAG-NEXT: s_clause 0x8
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[36:37] offset:80
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[36:37] offset:96
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[36:37] offset:48
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[36:37] offset:64
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[36:37] offset:16
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[36:37] offset:32
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[4:7], v[32:33] offset:48
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[8:11], v[32:33] offset:96
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[12:15], v[32:33] offset:64
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[16:19], v[32:33] offset:80
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[20:23], v[32:33] offset:16
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[24:27], v[32:33] offset:32
; GFX10-SDAG-NEXT: flat_load_dwordx4 v[28:31], v[0:1] offset:12
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[36:37]
-; GFX10-SDAG-NEXT: flat_load_dwordx4 v[36:39], v[36:37] offset:112
+; GFX10-SDAG-NEXT: flat_load_dwordx4 v[32:35], v[32:33]
+; GFX10-SDAG-NEXT: flat_load_dwordx3 v[36:38], v[36:37]
; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX10-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
; GFX10-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(8) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:48
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(7) lgkmcnt(8)
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(5)
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:48
; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[8:11] offset:64
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(6) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[12:15] offset:64
-; GFX10-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
-; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[16:19] offset:32
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[4:7] offset:64
+; GFX10-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[12:15] offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(4) lgkmcnt(8)
; GFX10-SDAG-NEXT: flat_store_dwordx4 v[2:3], v[20:23] offset:32
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(3) lgkmcnt(8)
@@ -638,7 +637,7 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX942-SDAG-NEXT: flat_load_dwordx4 v[52:55], v[0:1] offset:12
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x8c
; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942-SDAG-NEXT: flat_load_dwordx4 a[0:3], v[0:1]
+; GFX942-SDAG-NEXT: flat_load_dwordx3 a[0:2], v[0:1]
; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x60
; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x50
@@ -671,38 +670,45 @@ define void @flat_offset_inbounds_very_wide(ptr %p, ptr %pout, i32 %i) {
; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
-; GFX11-SDAG-NEXT: v_add_co_u32 v36, vcc_lo, v0, 28
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v1, vcc_lo
-; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[36:37] offset:80
-; GFX11-SDAG-NEXT: flat_load_b128 v[8:11], v[36:37] offset:96
-; GFX11-SDAG-NEXT: flat_load_b128 v[12:15], v[36:37] offset:64
-; GFX11-SDAG-NEXT: flat_load_b128 v[16:19], v[36:37] offset:32
-; GFX11-SDAG-NEXT: flat_load_b128 v[20:23], v[36:37] offset:16
-; GFX11-SDAG-NEXT: flat_load_b128 v[24:27], v[36:37]
+; GFX11-SDAG-NEXT: v_add_co_u32 v32, vcc_lo, v0, 28
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT: v_add_co_u32 v34, vcc_lo, 0x8c, v0
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v35, null, 0, v1, vcc_lo
+; GFX11-SDAG-NEXT: s_clause 0x8
+; GFX11-SDAG-NEXT: flat_load_b128 v[4:7], v[32:33] offset:96
+; GFX11-SDAG-NEXT: flat_load_b128 v[8:11], v[32:33] offset:64
+; GFX11-SDAG-NEXT: flat_load_b128 v[12:15], v[32:33] offset:80
+; GFX11-SDAG-NEXT: flat_load_b128 v[16:19], v[32:33] offset:32
+; GFX11-SDAG-NEXT: flat_load_b128 v[20:23], v[32:33] offset:16
+; GFX11-SDAG-NEXT: flat_load_b128 v[24:27], v[32:33]
; GFX11-SDAG-NEXT: flat_load_b128 v[28:31], v[0:1] offset:12
-; GFX11-SDAG-NEXT: flat_load_b128 v[32:35], v[36:37] offset:112
-; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT: flat_load_b128 v[35:38], v[36:37] offset:48
+; GFX11-SDAG-NEXT: flat_load_b96 v[36:38], v[34:35]
+; GFX11-SDAG-NEXT: flat_load_b128 v[32:35], v[32:33] offset:48
; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, v2, 48
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX11-SDAG-NEXT: v_add_co_u32 v48, vcc_lo, 0x88, v2
; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v49, null, 0, v3, vcc_lo
-; GFX11-SDAG-NEXT: s_clause 0x7
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:48
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:64
-; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:32
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(6) lgkmcnt(6)
+; GFX11-SDAG-NEXT: s_clause 0x2
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[12:15] offset:48
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[4:7] offset:64
+; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[8:11] offset:32
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(5) lgkmcnt(8)
; GFX11-SDAG-NEXT: flat_store_b128 v[0:1], v[16:19]
+; GFX11-SD...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/203484
More information about the llvm-commits
mailing list