[llvm] [SelectionDAG] Merge consecutive loads feeding as shuffle operands (PR #207303)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 21 02:35:30 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/207303
>From 5718442477c66f49ef9c7d4f9ba4c7f04499fe02 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Fri, 3 Jul 2026 03:41:46 -0700
Subject: [PATCH 01/10] [NFC] Add test for PR #207303
---
.../merge-loads-feeding-as-shuffleops.ll | 170 ++++++++++++++++++
1 file changed, 170 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
new file mode 100644
index 0000000000000..9ac340e51bd0e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=128 -mattr=+sve < %s | FileCheck %s --check-prefix=VBITS_GE_128
+; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve < %s | FileCheck %s --check-prefix=VBITS_GE_256
+; RUN: llc -mtriple=aarch64_be -aarch64-sve-vector-bits-min=256 -mattr=+sve < %s | FileCheck %s --check-prefix=BE_VBITS_GE_256
+
+define void @combine_loads_v16i8_v32i8(ptr %a) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
+ store <32 x i8> %result, ptr %a
+ ret void
+}
+
+define void @combine_loads_v16i8_v16i8_diff_load_chain(ptr %a, ptr %b) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v16i8_diff_load_chain:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldr q0, [x0]
+; VBITS_GE_128-NEXT: strb wzr, [x1]
+; VBITS_GE_128-NEXT: ldr q1, [x0, #16]
+; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v16i8_diff_load_chain:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldr q0, [x0]
+; VBITS_GE_256-NEXT: strb wzr, [x1]
+; VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; VBITS_GE_256-NEXT: ldr q1, [x0, #16]
+; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v16i8_diff_load_chain:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: strb wzr, [x1]
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ store i8 0, ptr %b
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
+ store <32 x i8> %result, ptr %a
+ ret void
+}
+
+define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; VBITS_GE_256-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_256-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l2, <16 x i8> %l1, <32 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
+ store <32 x i8> %result, ptr %a
+ ret void
+}
+
+define void @combine_loads_v8i16_v16i16(ptr %a) {
+; VBITS_GE_128-LABEL: combine_loads_v8i16_v16i16:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #4
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
+; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
+; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.8h }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl8
+; BE_VBITS_GE_256-NEXT: ld1 { v1.8h }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
+; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; BE_VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <8 x i16>, ptr %a, i64 1
+ %l1 = load <8 x i16>, ptr %a
+ %l2 = load <8 x i16>, ptr %gep
+ %result = shufflevector <8 x i16> %l1, <8 x i16> %l2, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+ store <16 x i16> %result, ptr %a
+ ret void
+}
>From 1ec89c29403e10cbdef9aee983ef33ea0bd5d4eb Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 2 Jul 2026 18:14:41 -0700
Subject: [PATCH 02/10] [SelectionDAG] Merge consecutive loads feeding as
shuffle operands
Perform the following fold given loadA and loadB can be proven consecutive:
concat(shuffle(mask0, loadA, loadB), shuffle(mask1, loadA, loadB))
-> shuffle(concat(mask0, mask1), loadAB, poison)
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 89 +++++++++++++++++++
.../merge-loads-feeding-as-shuffleops.ll | 65 +++++++-------
llvm/test/CodeGen/Thumb2/mve-vst2.ll | 22 ++---
3 files changed, 133 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e04e9578a3232..1790dc5fda804 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27007,6 +27007,92 @@ static SDValue combineConcatVectorOfShuffleAndItsOperands(
return DAG.getVectorShuffle(VT, dl, ShufOps[0], ShufOps[1], Mask);
}
+// concat(shuffle(mask0, loadA, loadB), shuffle(mask1, loadA, loadB))
+// -> shuffle(concat(mask0, mask1), loadAB, poison)
+// only if loadA and loadB can be proven consecutive.
+static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
+ const TargetLowering &TLI) {
+ SDValue A, B;
+ if (!sd_match(N,
+ m_Node(ISD::CONCAT_VECTORS, m_Shuffle(m_Value(A), m_Value(B)),
+ m_Shuffle(m_Deferred(A), m_Deferred(B)))))
+ return SDValue();
+ auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
+ auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
+ if (!L00 || !L01)
+ return SDValue();
+
+ // Check if the address spaces of both loads are the same.
+ if (L00->getAddressSpace() != L01->getAddressSpace())
+ return SDValue();
+
+ // Check if the wide load would be faster than the two separate loads.
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
+ L00->getMemoryVT().getVectorElementType(),
+ L00->getMemoryVT().getVectorNumElements() * 2);
+ unsigned Fast = 0;
+ Align NewAlign = L00->getAlign();
+ if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
+ L00->getAddressSpace(), NewAlign,
+ L00->getMemOperand()->getFlags(), &Fast) ||
+ !Fast)
+ return SDValue();
+
+ // Check if the loads are consecutive.
+ LoadSDNode *Base = nullptr;
+ LoadSDNode *Next = nullptr;
+ if (DAG.areNonVolatileConsecutiveLoads(
+ L01, L00, L01->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
+ Base = L00;
+ Next = L01;
+ } else if (DAG.areNonVolatileConsecutiveLoads(
+ L00, L01, L00->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
+ Base = L01;
+ Next = L00;
+ } else {
+ return SDValue(); // not adjacent
+ }
+
+ // Check if this is big endian target. If yes, we need to reverse the wide
+ // load order using bswap, which requires a scalar size that is a multiple
+ // of 16 bits.
+ bool NeedBSwap = DAG.getDataLayout().isBigEndian();
+ if (NeedBSwap && L00->getMemoryVT().getScalarSizeInBits() % 16 != 0)
+ return SDValue();
+
+ // Create a wide load of twice the size of the original load.
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineMemOperand *WideMMO = MF.getMachineMemOperand(
+ Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
+ SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
+ Base->getBasePtr(), WideMMO);
+ if (NeedBSwap) {
+ WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
+ }
+
+ // Create a shuffle of the wide load.
+ SmallVector<int, 32> Mask;
+ Mask.reserve(WideVT.getVectorNumElements());
+ auto *SV0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode());
+ auto *SV1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode());
+ ArrayRef<int> M0 = SV0->getMask();
+ ArrayRef<int> M1 = SV1->getMask();
+ if (Base == L00) {
+ llvm::append_range(Mask, M0);
+ llvm::append_range(Mask, M1);
+ } else {
+ int Sz = L00->getMemoryVT().getVectorNumElements();
+ for (unsigned I = 0; I < Sz; ++I)
+ Mask.push_back(M0[I] < Sz ? M0[I] + Sz : M0[I] - Sz);
+ for (unsigned I = 0; I < Sz; ++I)
+ Mask.push_back(M1[I] < Sz ? M1[I] + Sz : M1[I] - Sz);
+ }
+ // Create a new shuffle with the new mask.
+ SDValue NewShuffle = DAG.getVectorShuffle(WideVT, SDLoc(N), WideLoad,
+ DAG.getPOISON(WideVT), Mask);
+ return NewShuffle;
+}
+
static SDValue combineConcatVectorOfSplats(SDNode *N, SelectionDAG &DAG,
const TargetLowering &TLI,
bool LegalTypes,
@@ -27180,6 +27266,9 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
N, DAG, TLI, LegalTypes, LegalOperations))
return V;
+ if (SDValue V = combineConcatVectorOfShuffles(N, DAG, TLI))
+ return V;
+
// Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR
// nodes often generate nop CONCAT_VECTOR nodes. Scan the CONCAT_VECTOR
// operands and look for a CONCAT operations that place the incoming vectors
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
index 9ac340e51bd0e..eeafc1a7f3ff8 100644
--- a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -6,20 +6,20 @@
define void @combine_loads_v16i8_v32i8(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8:
; VBITS_GE_128: // %bb.0:
-; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #1
-; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #2
-; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: stp q0, q2, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
-; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
-; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: adrp x8, .LCPI0_0
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI0_0
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
@@ -93,19 +93,19 @@ define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: stp q0, q2, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; VBITS_GE_256-NEXT: ext v2.16b, v1.16b, v0.16b, #2
-; VBITS_GE_256-NEXT: ext v0.16b, v1.16b, v0.16b, #1
-; VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: adrp x8, .LCPI2_0
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI2_0
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
@@ -132,33 +132,32 @@ define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
define void @combine_loads_v8i16_v16i16(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v8i16_v16i16:
; VBITS_GE_128: // %bb.0:
-; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #2
-; VBITS_GE_128-NEXT: ext v0.16b, v0.16b, v1.16b, #4
-; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #4
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q0, q2, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
; VBITS_GE_256: // %bb.0:
-; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_256-NEXT: ptrue p0.h, vl8
-; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
-; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
-; VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
; VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; VBITS_GE_256-NEXT: adrp x8, .LCPI3_0
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI3_0
+; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x8]
+; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT: tbl z0.h, { z1.h }, z0.h
; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
; VBITS_GE_256-NEXT: ret
;
; BE_VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
; BE_VBITS_GE_256: // %bb.0:
-; BE_VBITS_GE_256-NEXT: add x8, x0, #16
-; BE_VBITS_GE_256-NEXT: ld1 { v0.8h }, [x0]
-; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl8
-; BE_VBITS_GE_256-NEXT: ld1 { v1.8h }, [x8]
-; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #4
-; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #2
-; BE_VBITS_GE_256-NEXT: splice z0.h, p0, z0.h, z2.h
; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl16
+; BE_VBITS_GE_256-NEXT: adrp x8, .LCPI3_0
+; BE_VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI3_0
+; BE_VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
+; BE_VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x8]
+; BE_VBITS_GE_256-NEXT: revb z0.h, p0/m, z0.h
+; BE_VBITS_GE_256-NEXT: tbl z0.h, { z0.h }, z1.h
; BE_VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
; BE_VBITS_GE_256-NEXT: ret
%gep = getelementptr <8 x i16>, ptr %a, i64 1
diff --git a/llvm/test/CodeGen/Thumb2/mve-vst2.ll b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
index fa7b0e4d6bcc8..2129ea7f3764e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vst2.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vst2.ll
@@ -6,11 +6,12 @@
define void @vst2_v2i32(ptr %src, ptr %dst) {
; CHECK-LABEL: vst2_v2i32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: ldm.w r0, {r2, r3, r12}
-; CHECK-NEXT: ldr r0, [r0, #12]
-; CHECK-NEXT: vmov q0[2], q0[0], r2, r3
-; CHECK-NEXT: vmov q0[3], q0[1], r12, r0
-; CHECK-NEXT: vstrw.32 q0, [r1]
+; CHECK-NEXT: vldrw.u32 q0, [r0]
+; CHECK-NEXT: vmov.f32 s4, s0
+; CHECK-NEXT: vmov.f32 s5, s2
+; CHECK-NEXT: vmov.f32 s6, s1
+; CHECK-NEXT: vmov.f32 s7, s3
+; CHECK-NEXT: vstrw.32 q1, [r1]
; CHECK-NEXT: bx lr
entry:
%l1 = load <2 x i32>, ptr %src, align 4
@@ -363,11 +364,12 @@ entry:
define void @vst2_v2f32(ptr %src, ptr %dst) {
; CHECK-LABEL: vst2_v2f32:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vldr s0, [r0]
-; CHECK-NEXT: vldr s2, [r0, #4]
-; CHECK-NEXT: vldr s1, [r0, #8]
-; CHECK-NEXT: vldr s3, [r0, #12]
-; CHECK-NEXT: vstrw.32 q0, [r1]
+; CHECK-NEXT: vldrw.u32 q0, [r0]
+; CHECK-NEXT: vmov.f32 s4, s0
+; CHECK-NEXT: vmov.f32 s5, s2
+; CHECK-NEXT: vmov.f32 s6, s1
+; CHECK-NEXT: vmov.f32 s7, s3
+; CHECK-NEXT: vstrw.32 q1, [r1]
; CHECK-NEXT: bx lr
entry:
%l1 = load <2 x float>, ptr %src, align 4
>From 2ac39c11f28dcc1c3d899883ee9b627553af1dd0 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Mon, 6 Jul 2026 23:13:38 -0700
Subject: [PATCH 03/10] address reviewer's comments
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 24 ++++++++-----------
1 file changed, 10 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 1790dc5fda804..10af0398e4284 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27007,15 +27007,16 @@ static SDValue combineConcatVectorOfShuffleAndItsOperands(
return DAG.getVectorShuffle(VT, dl, ShufOps[0], ShufOps[1], Mask);
}
-// concat(shuffle(mask0, loadA, loadB), shuffle(mask1, loadA, loadB))
-// -> shuffle(concat(mask0, mask1), loadAB, poison)
+// concat(shuffle(loadA, loadB, mask0), shuffle(loadA, loadB, mask1))
+// -> shuffle(loadAB, poison, concat(mask0, mask1))
// only if loadA and loadB can be proven consecutive.
static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
const TargetLowering &TLI) {
SDValue A, B;
- if (!sd_match(N,
- m_Node(ISD::CONCAT_VECTORS, m_Shuffle(m_Value(A), m_Value(B)),
- m_Shuffle(m_Deferred(A), m_Deferred(B)))))
+ ArrayRef<int> M0, M1;
+ if (!sd_match(N, m_Node(ISD::CONCAT_VECTORS,
+ m_Shuffle(m_Value(A), m_Value(B), m_Mask(M0)),
+ m_Shuffle(m_Deferred(A), m_Deferred(B), m_Mask(M1)))))
return SDValue();
auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
@@ -27027,9 +27028,9 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
return SDValue();
// Check if the wide load would be faster than the two separate loads.
- EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
- L00->getMemoryVT().getVectorElementType(),
- L00->getMemoryVT().getVectorNumElements() * 2);
+ EVT WideVT =
+ L00->getMemoryVT().getDoubleNumVectorElementsVT(*DAG.getContext());
+
unsigned Fast = 0;
Align NewAlign = L00->getAlign();
if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
@@ -27066,17 +27067,12 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
Base->getBasePtr(), WideMMO);
- if (NeedBSwap) {
+ if (NeedBSwap)
WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
- }
// Create a shuffle of the wide load.
SmallVector<int, 32> Mask;
Mask.reserve(WideVT.getVectorNumElements());
- auto *SV0 = cast<ShuffleVectorSDNode>(N->getOperand(0).getNode());
- auto *SV1 = cast<ShuffleVectorSDNode>(N->getOperand(1).getNode());
- ArrayRef<int> M0 = SV0->getMask();
- ArrayRef<int> M1 = SV1->getMask();
if (Base == L00) {
llvm::append_range(Mask, M0);
llvm::append_range(Mask, M1);
>From 72375ca2827f328e0e3672628a41a393091788b3 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Wed, 8 Jul 2026 04:41:32 -0700
Subject: [PATCH 04/10] add makeEquivalentMemoryOrdering calls
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 ++++++-----
.../merge-loads-feeding-as-shuffleops.ll | 18 +++++++++---------
2 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 10af0398e4284..8710357d21ca8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27041,15 +27041,12 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
// Check if the loads are consecutive.
LoadSDNode *Base = nullptr;
- LoadSDNode *Next = nullptr;
if (DAG.areNonVolatileConsecutiveLoads(
L01, L00, L01->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
Base = L00;
- Next = L01;
} else if (DAG.areNonVolatileConsecutiveLoads(
L00, L01, L00->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
Base = L01;
- Next = L00;
} else {
return SDValue(); // not adjacent
}
@@ -27067,6 +27064,10 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
Base->getBasePtr(), WideMMO);
+ SDValue WideMemOp = WideLoad;
+ // Redirect old chain users to the new chain.
+ DAG.makeEquivalentMemoryOrdering(L00, WideMemOp);
+ DAG.makeEquivalentMemoryOrdering(L01, WideMemOp);
if (NeedBSwap)
WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
@@ -27078,9 +27079,9 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
llvm::append_range(Mask, M1);
} else {
int Sz = L00->getMemoryVT().getVectorNumElements();
- for (unsigned I = 0; I < Sz; ++I)
+ for (int I = 0; I < Sz; ++I)
Mask.push_back(M0[I] < Sz ? M0[I] + Sz : M0[I] - Sz);
- for (unsigned I = 0; I < Sz; ++I)
+ for (int I = 0; I < Sz; ++I)
Mask.push_back(M1[I] < Sz ? M1[I] + Sz : M1[I] - Sz);
}
// Create a new shuffle with the new mask.
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
index eeafc1a7f3ff8..3644072cbdd69 100644
--- a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -7,9 +7,9 @@ define void @combine_loads_v16i8_v32i8(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
-; VBITS_GE_128-NEXT: stp q0, q2, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
@@ -93,9 +93,9 @@ define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #1
-; VBITS_GE_128-NEXT: stp q0, q2, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
@@ -133,9 +133,9 @@ define void @combine_loads_v8i16_v16i16(ptr %a) {
; VBITS_GE_128-LABEL: combine_loads_v8i16_v16i16:
; VBITS_GE_128: // %bb.0:
; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
-; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #4
-; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
-; VBITS_GE_128-NEXT: stp q0, q2, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #4
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
; VBITS_GE_128-NEXT: ret
;
; VBITS_GE_256-LABEL: combine_loads_v8i16_v16i16:
>From 5d52ed1ad919ed978cb6ac53ef94b35a7eec0776 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Wed, 8 Jul 2026 09:32:46 -0700
Subject: [PATCH 05/10] add NON_EXTLoad checks, remove smallvector reserve
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8710357d21ca8..2ba9245ced0d8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27020,7 +27020,7 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
return SDValue();
auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
- if (!L00 || !L01)
+ if (!L00 || !L01 || !ISD::isNON_EXTLoad(L00) || !ISD::isNON_EXTLoad(L01))
return SDValue();
// Check if the address spaces of both loads are the same.
@@ -27073,7 +27073,6 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
// Create a shuffle of the wide load.
SmallVector<int, 32> Mask;
- Mask.reserve(WideVT.getVectorNumElements());
if (Base == L00) {
llvm::append_range(Mask, M0);
llvm::append_range(Mask, M1);
>From 987067b00a01e48bd33460ea21c3821d064638f4 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 9 Jul 2026 22:21:17 -0700
Subject: [PATCH 06/10] address reviewer comments
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 25 +++++++++++--------
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 2ba9245ced0d8..87140883dd232 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27014,9 +27014,12 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
const TargetLowering &TLI) {
SDValue A, B;
ArrayRef<int> M0, M1;
- if (!sd_match(N, m_Node(ISD::CONCAT_VECTORS,
- m_Shuffle(m_Value(A), m_Value(B), m_Mask(M0)),
- m_Shuffle(m_Deferred(A), m_Deferred(B), m_Mask(M1)))))
+ if (!sd_match(N,
+ m_Node(ISD::CONCAT_VECTORS,
+ m_OneUse(m_Shuffle(m_NUses<2>(m_Value(A)),
+ m_NUses<2>(m_Value(B)), m_Mask(M0))),
+ m_OneUse(m_Shuffle(m_Deferred(A), m_Deferred(B),
+ m_Mask(M1))))))
return SDValue();
auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
@@ -27031,14 +27034,6 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
EVT WideVT =
L00->getMemoryVT().getDoubleNumVectorElementsVT(*DAG.getContext());
- unsigned Fast = 0;
- Align NewAlign = L00->getAlign();
- if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
- L00->getAddressSpace(), NewAlign,
- L00->getMemOperand()->getFlags(), &Fast) ||
- !Fast)
- return SDValue();
-
// Check if the loads are consecutive.
LoadSDNode *Base = nullptr;
if (DAG.areNonVolatileConsecutiveLoads(
@@ -27051,6 +27046,14 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
return SDValue(); // not adjacent
}
+ unsigned Fast = 0;
+ Align NewAlign = L00->getAlign();
+ if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
+ Base->getAddressSpace(), NewAlign,
+ Base->getMemOperand()->getFlags(), &Fast) ||
+ !Fast)
+ return SDValue();
+
// Check if this is big endian target. If yes, we need to reverse the wide
// load order using bswap, which requires a scalar size that is a multiple
// of 16 bits.
>From 9225217fa525dae15202230d56faa0411929a37d Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Thu, 9 Jul 2026 22:24:52 -0700
Subject: [PATCH 07/10] address reviewer comments
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 87140883dd232..333d25b1fcf8f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27047,7 +27047,7 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
}
unsigned Fast = 0;
- Align NewAlign = L00->getAlign();
+ Align NewAlign = Base->getAlign();
if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
Base->getAddressSpace(), NewAlign,
Base->getMemOperand()->getFlags(), &Fast) ||
>From 78afbfdbca12b1d6f344fdfe205b6c324c79df86 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Fri, 10 Jul 2026 07:43:09 -0700
Subject: [PATCH 08/10] address review comments
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 48 +++++------
.../merge-loads-feeding-as-shuffleops.ll | 84 +++++++++++++++++++
2 files changed, 107 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 333d25b1fcf8f..e89d7659e6471 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27021,33 +27021,33 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
m_OneUse(m_Shuffle(m_Deferred(A), m_Deferred(B),
m_Mask(M1))))))
return SDValue();
- auto *L00 = dyn_cast<LoadSDNode>(A.getNode());
- auto *L01 = dyn_cast<LoadSDNode>(B.getNode());
- if (!L00 || !L01 || !ISD::isNON_EXTLoad(L00) || !ISD::isNON_EXTLoad(L01))
+ auto *LoadA = dyn_cast<LoadSDNode>(A.getNode());
+ auto *LoadB = dyn_cast<LoadSDNode>(B.getNode());
+ if (!LoadA || !LoadB || !ISD::isNON_EXTLoad(LoadA) ||
+ !ISD::isNON_EXTLoad(LoadB))
return SDValue();
// Check if the address spaces of both loads are the same.
- if (L00->getAddressSpace() != L01->getAddressSpace())
+ if (LoadA->getAddressSpace() != LoadB->getAddressSpace())
return SDValue();
- // Check if the wide load would be faster than the two separate loads.
- EVT WideVT =
- L00->getMemoryVT().getDoubleNumVectorElementsVT(*DAG.getContext());
-
// Check if the loads are consecutive.
LoadSDNode *Base = nullptr;
if (DAG.areNonVolatileConsecutiveLoads(
- L01, L00, L01->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
- Base = L00;
+ LoadB, LoadA, LoadB->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
+ Base = LoadA;
} else if (DAG.areNonVolatileConsecutiveLoads(
- L00, L01, L00->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
- Base = L01;
+ LoadA, LoadB, LoadA->getMemoryVT().getStoreSize(),
+ /*Dist=*/1)) {
+ Base = LoadB;
} else {
return SDValue(); // not adjacent
}
unsigned Fast = 0;
Align NewAlign = Base->getAlign();
+ EVT WideVT =
+ LoadA->getMemoryVT().getDoubleNumVectorElementsVT(*DAG.getContext());
if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), WideVT,
Base->getAddressSpace(), NewAlign,
Base->getMemOperand()->getFlags(), &Fast) ||
@@ -27058,7 +27058,7 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
// load order using bswap, which requires a scalar size that is a multiple
// of 16 bits.
bool NeedBSwap = DAG.getDataLayout().isBigEndian();
- if (NeedBSwap && L00->getMemoryVT().getScalarSizeInBits() % 16 != 0)
+ if (NeedBSwap && LoadA->getMemoryVT().getScalarSizeInBits() % 16 != 0)
return SDValue();
// Create a wide load of twice the size of the original load.
@@ -27067,29 +27067,27 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
Base->getBasePtr(), WideMMO);
- SDValue WideMemOp = WideLoad;
// Redirect old chain users to the new chain.
- DAG.makeEquivalentMemoryOrdering(L00, WideMemOp);
- DAG.makeEquivalentMemoryOrdering(L01, WideMemOp);
+ DAG.makeEquivalentMemoryOrdering(LoadA, WideLoad);
+ DAG.makeEquivalentMemoryOrdering(LoadB, WideLoad);
if (NeedBSwap)
WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
// Create a shuffle of the wide load.
SmallVector<int, 32> Mask;
- if (Base == L00) {
+ if (Base == LoadA) {
llvm::append_range(Mask, M0);
llvm::append_range(Mask, M1);
} else {
- int Sz = L00->getMemoryVT().getVectorNumElements();
- for (int I = 0; I < Sz; ++I)
- Mask.push_back(M0[I] < Sz ? M0[I] + Sz : M0[I] - Sz);
- for (int I = 0; I < Sz; ++I)
- Mask.push_back(M1[I] < Sz ? M1[I] + Sz : M1[I] - Sz);
+ SmallVector<int, 16> C0(M0), C1(M1);
+ ShuffleVectorSDNode::commuteMask(C0);
+ ShuffleVectorSDNode::commuteMask(C1);
+ llvm::append_range(Mask, C0);
+ llvm::append_range(Mask, C1);
}
// Create a new shuffle with the new mask.
- SDValue NewShuffle = DAG.getVectorShuffle(WideVT, SDLoc(N), WideLoad,
- DAG.getPOISON(WideVT), Mask);
- return NewShuffle;
+ return DAG.getVectorShuffle(WideVT, SDLoc(N), WideLoad, DAG.getPOISON(WideVT),
+ Mask);
}
static SDValue combineConcatVectorOfSplats(SDNode *N, SelectionDAG &DAG,
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
index 3644072cbdd69..ede5a51652901 100644
--- a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -167,3 +167,87 @@ define void @combine_loads_v8i16_v16i16(ptr %a) {
store <16 x i16> %result, ptr %a
ret void
}
+
+define void @combine_loads_v16i8_v16i8_multi_use_load(ptr %a, ptr %b) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v16i8_multi_use_load:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v0.16b, v1.16b, #1
+; VBITS_GE_128-NEXT: ext v1.16b, v0.16b, v1.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q1, [x0]
+; VBITS_GE_128-NEXT: str q0, [x1]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v16i8_multi_use_load:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ldp q0, q1, [x0]
+; VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v1.16b, #1
+; VBITS_GE_256-NEXT: splice z1.b, p0, z1.b, z2.b
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; VBITS_GE_256-NEXT: str q0, [x1]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v16i8_multi_use_load:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v1.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z1.b, p0, z1.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: st1 { v0.16b }, [x1]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
+ store <32 x i8> %result, ptr %a
+ store <16 x i8> %l1, ptr %b
+ ret void
+}
+
+define void @combine_loads_v16i8_v32i8_shuffle_with_poison_mask_indices(ptr %a, ptr %b) {
+; VBITS_GE_128-LABEL: combine_loads_v16i8_v32i8_shuffle_with_poison_mask_indices:
+; VBITS_GE_128: // %bb.0:
+; VBITS_GE_128-NEXT: ldp q1, q0, [x0]
+; VBITS_GE_128-NEXT: ext v2.16b, v1.16b, v0.16b, #1
+; VBITS_GE_128-NEXT: ext v0.16b, v1.16b, v0.16b, #2
+; VBITS_GE_128-NEXT: stp q2, q0, [x0]
+; VBITS_GE_128-NEXT: ret
+;
+; VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_shuffle_with_poison_mask_indices:
+; VBITS_GE_256: // %bb.0:
+; VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; VBITS_GE_256-NEXT: adrp x8, .LCPI5_0
+; VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI5_0
+; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
+; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; VBITS_GE_256-NEXT: ret
+;
+; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_shuffle_with_poison_mask_indices:
+; BE_VBITS_GE_256: // %bb.0:
+; BE_VBITS_GE_256-NEXT: add x8, x0, #16
+; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
+; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
+; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
+; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
+; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
+; BE_VBITS_GE_256-NEXT: ret
+ %gep = getelementptr <16 x i8>, ptr %a, i64 1
+ %l1 = load <16 x i8>, ptr %a
+ %l2 = load <16 x i8>, ptr %gep
+ %result = shufflevector <16 x i8> %l1, <16 x i8> %l2, <32 x i32><i32 1, i32 2, i32 poison, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16,i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 poison, i32 15, i32 poison, i32 17>
+ store <32 x i8> %result, ptr %a
+ ret void
+}
>From d4e59c747a0a4be6e38624781ac16136c9544bf0 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Fri, 10 Jul 2026 08:16:48 -0700
Subject: [PATCH 09/10] address big endian comments
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 ----
.../merge-loads-feeding-as-shuffleops.ll | 43 ++++++++-----------
2 files changed, 18 insertions(+), 34 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e89d7659e6471..b3ddd0c5f6961 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27054,13 +27054,6 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
!Fast)
return SDValue();
- // Check if this is big endian target. If yes, we need to reverse the wide
- // load order using bswap, which requires a scalar size that is a multiple
- // of 16 bits.
- bool NeedBSwap = DAG.getDataLayout().isBigEndian();
- if (NeedBSwap && LoadA->getMemoryVT().getScalarSizeInBits() % 16 != 0)
- return SDValue();
-
// Create a wide load of twice the size of the original load.
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *WideMMO = MF.getMachineMemOperand(
@@ -27070,8 +27063,6 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
// Redirect old chain users to the new chain.
DAG.makeEquivalentMemoryOrdering(LoadA, WideLoad);
DAG.makeEquivalentMemoryOrdering(LoadB, WideLoad);
- if (NeedBSwap)
- WideLoad = DAG.getNode(ISD::BSWAP, SDLoc(N), WideVT, WideLoad);
// Create a shuffle of the wide load.
SmallVector<int, 32> Mask;
diff --git a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
index ede5a51652901..9cfd380719853 100644
--- a/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
+++ b/llvm/test/CodeGen/AArch64/merge-loads-feeding-as-shuffleops.ll
@@ -25,14 +25,12 @@ define void @combine_loads_v16i8_v32i8(ptr %a) {
;
; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8:
; BE_VBITS_GE_256: // %bb.0:
-; BE_VBITS_GE_256-NEXT: add x8, x0, #16
-; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
-; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
-; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
-; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
-; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: adrp x8, .LCPI0_0
+; BE_VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI0_0
+; BE_VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; BE_VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; BE_VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
; BE_VBITS_GE_256-NEXT: ret
%gep = getelementptr <16 x i8>, ptr %a, i64 1
@@ -111,14 +109,12 @@ define void @combine_loads_v16i8_v32i8_diff_shuffle_order(ptr %a) {
;
; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_diff_shuffle_order:
; BE_VBITS_GE_256: // %bb.0:
-; BE_VBITS_GE_256-NEXT: add x8, x0, #16
-; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
-; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
-; BE_VBITS_GE_256-NEXT: ext v2.16b, v1.16b, v0.16b, #2
-; BE_VBITS_GE_256-NEXT: ext v0.16b, v1.16b, v0.16b, #1
-; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: adrp x8, .LCPI2_0
+; BE_VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI2_0
+; BE_VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; BE_VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; BE_VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
; BE_VBITS_GE_256-NEXT: ret
%gep = getelementptr <16 x i8>, ptr %a, i64 1
@@ -154,10 +150,9 @@ define void @combine_loads_v8i16_v16i16(ptr %a) {
; BE_VBITS_GE_256-NEXT: ptrue p0.h, vl16
; BE_VBITS_GE_256-NEXT: adrp x8, .LCPI3_0
; BE_VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI3_0
-; BE_VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]
-; BE_VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x8]
-; BE_VBITS_GE_256-NEXT: revb z0.h, p0/m, z0.h
-; BE_VBITS_GE_256-NEXT: tbl z0.h, { z0.h }, z1.h
+; BE_VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x8]
+; BE_VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]
+; BE_VBITS_GE_256-NEXT: tbl z0.h, { z1.h }, z0.h
; BE_VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0]
; BE_VBITS_GE_256-NEXT: ret
%gep = getelementptr <8 x i16>, ptr %a, i64 1
@@ -234,14 +229,12 @@ define void @combine_loads_v16i8_v32i8_shuffle_with_poison_mask_indices(ptr %a,
;
; BE_VBITS_GE_256-LABEL: combine_loads_v16i8_v32i8_shuffle_with_poison_mask_indices:
; BE_VBITS_GE_256: // %bb.0:
-; BE_VBITS_GE_256-NEXT: add x8, x0, #16
-; BE_VBITS_GE_256-NEXT: ld1 { v0.16b }, [x0]
-; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl16
-; BE_VBITS_GE_256-NEXT: ld1 { v1.16b }, [x8]
-; BE_VBITS_GE_256-NEXT: ext v2.16b, v0.16b, v1.16b, #2
-; BE_VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v1.16b, #1
-; BE_VBITS_GE_256-NEXT: splice z0.b, p0, z0.b, z2.b
; BE_VBITS_GE_256-NEXT: ptrue p0.b, vl32
+; BE_VBITS_GE_256-NEXT: adrp x8, .LCPI5_0
+; BE_VBITS_GE_256-NEXT: add x8, x8, :lo12:.LCPI5_0
+; BE_VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x8]
+; BE_VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
+; BE_VBITS_GE_256-NEXT: tbl z0.b, { z1.b }, z0.b
; BE_VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0]
; BE_VBITS_GE_256-NEXT: ret
%gep = getelementptr <16 x i8>, ptr %a, i64 1
>From 725bec9b3377351a6ec97675bf6be88deecaa92a Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Mon, 20 Jul 2026 22:38:43 -0700
Subject: [PATCH 10/10] Check if wide load , wide shuffle and its mask is legal
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 33 ++++++++++++-------
1 file changed, 21 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index b3ddd0c5f6961..578e126d0f96c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27011,7 +27011,8 @@ static SDValue combineConcatVectorOfShuffleAndItsOperands(
// -> shuffle(loadAB, poison, concat(mask0, mask1))
// only if loadA and loadB can be proven consecutive.
static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
- const TargetLowering &TLI) {
+ const TargetLowering &TLI,
+ bool LegalOperations) {
SDValue A, B;
ArrayRef<int> M0, M1;
if (!sd_match(N,
@@ -27054,16 +27055,6 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
!Fast)
return SDValue();
- // Create a wide load of twice the size of the original load.
- MachineFunction &MF = DAG.getMachineFunction();
- MachineMemOperand *WideMMO = MF.getMachineMemOperand(
- Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
- SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
- Base->getBasePtr(), WideMMO);
- // Redirect old chain users to the new chain.
- DAG.makeEquivalentMemoryOrdering(LoadA, WideLoad);
- DAG.makeEquivalentMemoryOrdering(LoadB, WideLoad);
-
// Create a shuffle of the wide load.
SmallVector<int, 32> Mask;
if (Base == LoadA) {
@@ -27076,6 +27067,24 @@ static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
llvm::append_range(Mask, C0);
llvm::append_range(Mask, C1);
}
+
+ // Check if the wide load, new shuffle and it's mask is legal.
+ if (LegalOperations &&
+ (!TLI.isOperationLegal(ISD::LOAD, WideVT) ||
+ !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, WideVT) ||
+ !TLI.isShuffleMaskLegal(Mask, WideVT)))
+ return SDValue();
+
+ // Create a wide load of twice the size of the original load.
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineMemOperand *WideMMO = MF.getMachineMemOperand(
+ Base->getMemOperand(), /*Offset=*/0, WideVT.getStoreSize());
+ SDValue WideLoad = DAG.getLoad(WideVT, SDLoc(N), Base->getChain(),
+ Base->getBasePtr(), WideMMO);
+ // Redirect old chain users to the new chain.
+ DAG.makeEquivalentMemoryOrdering(LoadA, WideLoad);
+ DAG.makeEquivalentMemoryOrdering(LoadB, WideLoad);
+
// Create a new shuffle with the new mask.
return DAG.getVectorShuffle(WideVT, SDLoc(N), WideLoad, DAG.getPOISON(WideVT),
Mask);
@@ -27254,7 +27263,7 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
N, DAG, TLI, LegalTypes, LegalOperations))
return V;
- if (SDValue V = combineConcatVectorOfShuffles(N, DAG, TLI))
+ if (SDValue V = combineConcatVectorOfShuffles(N, DAG, TLI, LegalOperations))
return V;
// Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR
More information about the llvm-commits
mailing list