[llvm] [SelectionDAG] Merge consecutive loads feeding as shuffle operands (PR #207303)
Sushant Gokhale via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 09:35:25 PDT 2026
https://github.com/sushgokh updated https://github.com/llvm/llvm-project/pull/207303
>From 5718442477c66f49ef9c7d4f9ba4c7f04499fe02 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Fri, 3 Jul 2026 03:41:46 -0700
Subject: [PATCH 1/5] [NFC] Add test for PR #207303
---
.../merge-loads-feeding-as-shuffleops.ll | 170 ++++++++++++++++++
1 file changed, 170 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
new file mode 100644
index 0000000000000..9ac340e51bd0e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=128 -mattr=+sve < %s | FileCheck %s --check-prefix=VBITS_GE_128
+; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve < %s | FileCheck %s --check-prefix=VBITS_GE_256
+; RUN: llc -mtriple=aarch64_be -aarch64-sve-vector-bits-min=256 -mattr=+sve < %s | FileCheck %s --check-prefix=BE_VBITS_GE_256
+
+define void @combine_loads_v16i8_v32i8(ptr %a) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
+ store <32 x i8> %result, ptr %a
+ ret void
+}
+
+define void @combine_loads_v16i8_v16i8_diff_load_chain(ptr %a, ptr %b) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v16i8_diff_load_chain:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldr q0, [x0]
+; VBITS_GE_128-NEXT: strb wzr, [x1]
+; VBITS_GE_128-NEXT: ldr q1, [x0, #16]
+; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v16i8_diff_load_chain:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldr q0, [x0]
+; VBITS_GE_256-NEXT: strb wzr, [x1]
+; VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; VBITS_GE_256-NEXT: ldr q1, [x0, #16]
+; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v16i8_diff_load_chain:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: strb wzr, [x1]
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ store i8 0, ptr %b
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
+ store <32 x i8> %result, ptr %a
+ ret void
+}
+
+define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; VBITS_GE_256-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_256-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l2, <16 x i8> %l1, <32 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
+ store <32 x i8> %result, ptr %a
+ ret void
+}
+
+define void @combine_loads_v8i16_v16i16(ptr %a) {
+; VBITS_GE_128-LABEL: combine_loads_v8i16_v16i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #4
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
+; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.8h }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; BE_VBITS_GE_256-NEXT: ld1 { v1.8h }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
+; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; BE_VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <8 x i16>, ptr %a, i64 1
+ %l1 = load <8 x i16>, ptr %a
+ %l2 = load <8 x i16>, ptr %gep
+ %result = shufflevector <8 x i16> %l1, <8 x i16> %l2, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+ store <16 x i16> %result, ptr %a
+ ret void
+}
>From 1ec89c29403e10cbdef9aee983ef33ea0bd5d4eb Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 2 Jul 2026 18:14:41 -0700
Subject: [PATCH 2/5] [SelectionDAG] Merge consecutive loads feeding as shuffle
operands
Perform the following fold given loadA and loadB can be proven consecutive:
concat(shuffle(mask0, loadA, loadB), shuffle(mask1, loadA, loadB))
-> shuffle(concat(mask0, mask1), loadAB, poison)
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 89 +++++++++++++++++++
.../merge-loads-feeding-as-shuffleops.ll | 65 +++++++-------
llvm/test/CodeGen/Thumb2/mve-vst2.ll | 22 ++---
3 files changed, 133 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e04e9578a3232..1790dc5fda804 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27007,6 +27007,92 @@ static SDValue combineConcatVectorOfShuffleAndItsOperands(
return DAG.getVectorShuffle(VT, dl, ShufOps[0], ShufOps[1], Mask);
}
+// concat(shuffle(mask0, loadA, loadB), shuffle(mask1, loadA, loadB))
+// -> shuffle(concat(mask0, mask1), loadAB, poison)
+// only if loadA and loadB can be proven consecutive.
+static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
+ const TargetLowering &TLI) {
+ SDValue A, B;
+ if (!sd_match(N,
+ m_Node(ISD::CONCAT_VECTORS, m_Shuffle(m_Value(A), m_Value(B)),
+ m_Shuffle(m_Deferred(A), m_Deferred(B)))))
+ return SDValue();
+ auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
+ auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
+ if (!L00 || !L01)
+ return SDValue();
+
+ // Check if the address spaces of both loads are the same.
+ if (L00->getAddressSpace() != L01->getAddressSpace())
+ return SDValue();
+
+ // Check if the wide load would be faster than the two separate loads.
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
+ L00->getMemoryVT().getVectorElementType(),
+ L00->getMemoryVT().getVectorNumElements() * 2);
+ unsigned Fast = 0;
+ Align NewAlign = L00->getAlign();
+ if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
+ L00->getAddressSpace(), NewAlign,
+ L00->getMemOperand()->getFlags(), &Fast) ||
+ !Fast)
+ return SDValue();
+
+ // Check if the loads are consecutive.
+ LoadSDNode *Base = nullptr;
+ LoadSDNode *Next = nullptr;
+ if (DAG.areNonVolatileConsecutiveLoads(
+ L01, L00, L01->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
+ Base = L00;
+ Next = L01;
+ } else if (DAG.areNonVolatileConsecutiveLoads(
+ L00, L01, L00->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
+ Base = L01;
+ Next = L00;
+ } else {
+ return SDValue(); // not adjacent
+ }
+
+ // Check if this is big endian target. If yes, we need to reverse the wide
+ // load order using bswap, which requires a scalar size that is a multiple
+ // of 16 bits.
+ bool NeedBSwap = DAG.getDataLayout().isBigEndian();
+ if (NeedBSwap && L00->getMemoryVT().getScalarSizeInBits() % 16 != 0)
+ return SDValue();
+
+ // Create a wide load of twice the size of the original load.
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineMemOperand *WideMMO = MF.getMachineMemOperand(
+ Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
+ SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
+ Base->getBasePtr(), WideMMO);
+ if (NeedBSwap) {
+ WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
+ }
+
+ // Create a shuffle of the wide load.
+ SmallVector<int, 32> Mask;
+ Mask.reserve(WideVT.getVectorNumElements());
+ auto *SV0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode());
+ auto *SV1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode());
+ ArrayRef<int> M0 = SV0->getMask();
+ ArrayRef<int> M1 = SV1->getMask();
+ if (Base == L00) {
+ llvm::append_range(Mask, M0);
+ llvm::append_range(Mask, M1);
+ } else {
+ int Sz = L00->getMemoryVT().getVectorNumElements();
+ for (unsigned I = 0; I < Sz; ++I)
+ Mask.push_back(M0[I] < Sz ? M0[I] + Sz : M0[I] - Sz);
+ for (unsigned I = 0; I < Sz; ++I)
+ Mask.push_back(M1[I] < Sz ? M1[I] + Sz : M1[I] - Sz);
+ }
+ // Create a new shuffle with the new mask.
+ SDValue NewShuffle = DAG.getVectorShuffle(WideVT, SDLoc(N), WideLoad,
+ DAG.getPOISON(WideVT), Mask);
+ return NewShuffle;
+}
+
static SDValue combineConcatVectorOfSplats(SDNode *N, SelectionDAG &DAG,
const TargetLowering &TLI,
bool LegalTypes,
@@ -27180,6 +27266,9 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
N, DAG, TLI, LegalTypes, LegalOperations))
return V;
+ if (SDValue V = combineConcatVectorOfShuffles(N, DAG, TLI))
+ return V;
+
// Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR
// nodes often generate nop CONCAT_VECTOR nodes. Scan the CONCAT_VECTOR
// operands and look for a CONCAT operations that place the incoming vectors
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
index 9ac340e51bd0e..eeafc1a7f3ff8 100644
--- a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -6,20 +6,20 @@
define void @combine_loads_v16i8_v32i8(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8:
; VBITS_GE_128: // %bb.0:
-; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #1
-; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #2
-; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: stp q0, q2, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
-; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
-; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: adrp x8, .LCPI0_0
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI0_0
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
@@ -93,19 +93,19 @@ define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: stp q0, q2, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; VBITS_GE_256-NEXT: ext v2.16b, v1.16b, v0.16b, #2
-; VBITS_GE_256-NEXT: ext v0.16b, v1.16b, v0.16b, #1
-; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: adrp x8, .LCPI2_0
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI2_0
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
@@ -132,33 +132,32 @@ define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
define void @combine_loads_v8i16_v16i16(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v8i16_v16i16:
; VBITS_GE_128: // %bb.0:
-; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #2
-; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #4
-; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #4
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q0, q2, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.h, vl8
-; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
-; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
-; VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: adrp x8, .LCPI3_0
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI3_0
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x8]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: tbl z0.h, { z1.h }, z0.h
; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; BE_VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
; BE_VBITS_GE_256: // %bb.0:
-; BE_VBITS_GE_256-NEXT: add x8, x0, #16
-; BE_VBITS_GE_256-NEXT: ld1 { v0.8h }, [x0]
-; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl8
-; BE_VBITS_GE_256-NEXT: ld1 { v1.8h }, [x8]
-; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
-; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
-; BE_VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; BE_VBITS_GE_256-NEXT: adrp x8, .LCPI3_0
+; BE_VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI3_0
+; BE_VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
+; BE_VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x8]
+; BE_VBITS_GE_256-NEXT: revb z0.h, p0/m, z0.h
+; BE_VBITS_GE_256-NEXT: tbl z0.h, { z0.h }, z1.h
; BE_VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
; BE_VBITS_GE_256-NEXT: ret
%gep = getelementptr <8 x i16>, ptr %a, i64 1
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst2.ll b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
index fa7b0e4d6bcc8..2129ea7f3764e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vst2.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
@@ -6,11 +6,12 @@
define void @vst2_v2i32(ptr %src, ptr %dst) {
; CHECK-LABEL: vst2_v2i32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: ldm.w r0, {r2, r3, r12}
-; CHECK-NEXT: ldr r0, [r0, #12]
-; CHECK-NEXT: vmov q0[2], q0[0], r2, r3
-; CHECK-NEXT: vmov q0[3], q0[1], r12, r0
-; CHECK-NEXT: vstrw.32 q0, [r1]
+; CHECK-NEXT: vldrw.u32 q0, [r0]
+; CHECK-NEXT: vmov.f32 s4, s0
+; CHECK-NEXT: vmov.f32 s5, s2
+; CHECK-NEXT: vmov.f32 s6, s1
+; CHECK-NEXT: vmov.f32 s7, s3
+; CHECK-NEXT: vstrw.32 q1, [r1]
; CHECK-NEXT: bx lr
entry:
%l1 = load <2 x i32>, ptr %src, align 4
@@ -363,11 +364,12 @@ entry:
define void @vst2_v2f32(ptr %src, ptr %dst) {
; CHECK-LABEL: vst2_v2f32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vldr s0, [r0]
-; CHECK-NEXT: vldr s2, [r0, #4]
-; CHECK-NEXT: vldr s1, [r0, #8]
-; CHECK-NEXT: vldr s3, [r0, #12]
-; CHECK-NEXT: vstrw.32 q0, [r1]
+; CHECK-NEXT: vldrw.u32 q0, [r0]
+; CHECK-NEXT: vmov.f32 s4, s0
+; CHECK-NEXT: vmov.f32 s5, s2
+; CHECK-NEXT: vmov.f32 s6, s1
+; CHECK-NEXT: vmov.f32 s7, s3
+; CHECK-NEXT: vstrw.32 q1, [r1]
; CHECK-NEXT: bx lr
entry:
%l1 = load <2 x float>, ptr %src, align 4
>From 2ac39c11f28dcc1c3d899883ee9b627553af1dd0 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Mon, 6 Jul 2026 23:13:38 -0700
Subject: [PATCH 3/5] address reviewer's comments
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 24 ++++++++-----------
1 file changed, 10 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 1790dc5fda804..10af0398e4284 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27007,15 +27007,16 @@ static SDValue combineConcatVectorOfShuffleAndItsOperands(
return DAG.getVectorShuffle(VT, dl, ShufOps[0], ShufOps[1], Mask);
}
-// concat(shuffle(mask0, loadA, loadB), shuffle(mask1, loadA, loadB))
-// -> shuffle(concat(mask0, mask1), loadAB, poison)
+// concat(shuffle(loadA, loadB, mask0), shuffle(loadA, loadB, mask1))
+// -> shuffle(loadAB, poison, concat(mask0, mask1))
// only if loadA and loadB can be proven consecutive.
static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
const TargetLowering &TLI) {
SDValue A, B;
- if (!sd_match(N,
- m_Node(ISD::CONCAT_VECTORS, m_Shuffle(m_Value(A), m_Value(B)),
- m_Shuffle(m_Deferred(A), m_Deferred(B)))))
+ ArrayRef<int> M0, M1;
+ if (!sd_match(N, m_Node(ISD::CONCAT_VECTORS,
+ m_Shuffle(m_Value(A), m_Value(B), m_Mask(M0)),
+ m_Shuffle(m_Deferred(A), m_Deferred(B), m_Mask(M1)))))
return SDValue();
auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
@@ -27027,9 +27028,9 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
return SDValue();
// Check if the wide load would be faster than the two separate loads.
- EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
- L00->getMemoryVT().getVectorElementType(),
- L00->getMemoryVT().getVectorNumElements() * 2);
+ EVT WideVT =
+ L00->getMemoryVT().getDoubleNumVectorElementsVT(*DAG.getContext());
+
unsigned Fast = 0;
Align NewAlign = L00->getAlign();
if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
@@ -27066,17 +27067,12 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
Base->getBasePtr(), WideMMO);
- if (NeedBSwap) {
+ if (NeedBSwap)
WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
- }
// Create a shuffle of the wide load.
SmallVector<int, 32> Mask;
Mask.reserve(WideVT.getVectorNumElements());
- auto *SV0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode());
- auto *SV1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode());
- ArrayRef<int> M0 = SV0->getMask();
- ArrayRef<int> M1 = SV1->getMask();
if (Base == L00) {
llvm::append_range(Mask, M0);
llvm::append_range(Mask, M1);
>From 72375ca2827f328e0e3672628a41a393091788b3 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Wed, 8 Jul 2026 04:41:32 -0700
Subject: [PATCH 4/5] add makeEquivalentMemoryOrdering calls
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 ++++++-----
.../merge-loads-feeding-as-shuffleops.ll | 18 +++++++++---------
2 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 10af0398e4284..8710357d21ca8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27041,15 +27041,12 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
// Check if the loads are consecutive.
LoadSDNode *Base = nullptr;
- LoadSDNode *Next = nullptr;
if (DAG.areNonVolatileConsecutiveLoads(
L01, L00, L01->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
Base = L00;
- Next = L01;
} else if (DAG.areNonVolatileConsecutiveLoads(
L00, L01, L00->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
Base = L01;
- Next = L00;
} else {
return SDValue(); // not adjacent
}
@@ -27067,6 +27064,10 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
Base->getBasePtr(), WideMMO);
+ SDValue WideMemOp = WideLoad;
+ // Redirect old chain users to the new chain.
+ DAG.makeEquivalentMemoryOrdering(L00, WideMemOp);
+ DAG.makeEquivalentMemoryOrdering(L01, WideMemOp);
if (NeedBSwap)
WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
@@ -27078,9 +27079,9 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
llvm::append_range(Mask, M1);
} else {
int Sz = L00->getMemoryVT().getVectorNumElements();
- for (unsigned I = 0; I < Sz; ++I)
+ for (int I = 0; I < Sz; ++I)
Mask.push_back(M0[I] < Sz ? M0[I] + Sz : M0[I] - Sz);
- for (unsigned I = 0; I < Sz; ++I)
+ for (int I = 0; I < Sz; ++I)
Mask.push_back(M1[I] < Sz ? M1[I] + Sz : M1[I] - Sz);
}
// Create a new shuffle with the new mask.
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
index eeafc1a7f3ff8..3644072cbdd69 100644
--- a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -7,9 +7,9 @@ define void @combine_loads_v16i8_v32i8(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
-; VBITS_GE_128-NEXT: stp q0, q2, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
@@ -93,9 +93,9 @@ define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
-; VBITS_GE_128-NEXT: stp q0, q2, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
@@ -133,9 +133,9 @@ define void @combine_loads_v8i16_v16i16(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v8i16_v16i16:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #4
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: stp q0, q2, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #4
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
>From 5d52ed1ad919ed978cb6ac53ef94b35a7eec0776 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Wed, 8 Jul 2026 09:32:46 -0700
Subject: [PATCH 5/5] add NON_EXTLoad checks, remove smallvector reserve
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8710357d21ca8..2ba9245ced0d8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27020,7 +27020,7 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
return SDValue();
auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
- if (!L00 || !L01)
+ if (!L00 || !L01 || !ISD::isNON_EXTLoad(L00) || !ISD::isNON_EXTLoad(L01))
return SDValue();
// Check if the address spaces of both loads are the same.
@@ -27073,7 +27073,6 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
// Create a shuffle of the wide load.
SmallVector<int, 32> Mask;
- Mask.reserve(WideVT.getVectorNumElements());
if (Base == L00) {
llvm::append_range(Mask, M0);
llvm::append_range(Mask, M1);
More information about the llvm-commits
mailing list