[llvm] [AArch64] Transform interleave store to tbl for narrow vector types (PR #212224)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 01:48:42 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/212224
>From e6e3f935a320ac9ded623b27e7234922ffd2e7d4 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Fri, 24 Jul 2026 17:43:02 +0000
Subject: [PATCH 1/2] [AArch64] Add interleaved intrinsic test forms
---
llvm/test/CodeGen/AArch64/tbl-loops.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index 4870243e0e4a4..2e1578bb320fe 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -711,6 +711,8 @@ for.body: ; preds = %for.body.preheader4
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+
+
define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
; CHECK-IAENABLED-LABEL: loop4:
; CHECK-IAENABLED: // %bb.0: // %entry
@@ -1315,7 +1317,6 @@ for.body: ; preds = %for.body.preheader3
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-
define void @loop3_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
; CHECK-LABEL: loop3_intrinsic:
; CHECK: // %bb.0: // %entry
@@ -1533,7 +1534,6 @@ for.body: ; preds = %for.body.preheader4
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
-
define void @loop4_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr nocapture noundef readonly %data, i32 noundef %width) {
; CHECK-LABEL: loop4_intrinsic:
; CHECK: // %bb.0: // %entry
>From e43ae30c720f27e8d36bec19107d6c3e37245c60 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Fri, 24 Jul 2026 17:58:51 +0100
Subject: [PATCH 2/2] [AArch64] Use tbl for narrow vector types
---
.../Target/AArch64/AArch64ISelLowering.cpp | 71 +++++++++++++++
llvm/test/CodeGen/AArch64/tbl-loops.ll | 86 ++++++++-----------
2 files changed, 108 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2ac6c5fbc471a..d8331e64ee166 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27504,6 +27504,9 @@ static SDValue
performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG);
+static SDValue performNarrowInterleavedStoreCombine(
+ SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG);
+
static SDValue performSTORECombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
@@ -27535,6 +27538,9 @@ static SDValue performSTORECombine(SDNode *N,
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
+ if (SDValue Res = performNarrowInterleavedStoreCombine(N, DCI, DAG))
+ return Res;
+
if (SDValue Res = performInterleavedStoreCombine(N, DCI, DAG))
return Res;
@@ -27760,6 +27766,71 @@ static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
WideMask->getOperand(0));
}
+// Turn store(concat(vector_interleave(v0..vN-1))) into a store of a wide
+// BUILD_VECTOR that gathers the interleaved lanes when the sub-vector type is
+// too narrow for a structured stN store.
+// Type legalization widens the narrow inputs to NEON-sized vectors and
+// LowerBUILD_VECTOR can reconstruct the gather as one or more table lookups.
+static SDValue performNarrowInterleavedStoreCombine(
+ SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
+ if (!DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable())
+ return SDValue();
+
+ if (!DCI.isBeforeLegalize())
+ return SDValue();
+
+ StoreSDNode *ST = cast<StoreSDNode>(N);
+ if (!ISD::isNormalStore(ST) || !ST->isSimple() || !ST->getOffset().isUndef())
+ return SDValue();
+
+ SDValue Value = ST->getValue();
+ if (!Value.hasOneUse())
+ return SDValue();
+
+ SmallVector<SDValue, 4> Inputs;
+ if (!isSequentialConcatOfVectorInterleave(Value.getNode(), Inputs))
+ return SDValue();
+
+ unsigned Factor = Inputs.size();
+ if (Factor != 2 && Factor != 3 && Factor != 4)
+ return SDValue();
+
+ EVT SubVT = Inputs[0].getValueType();
+ if (!SubVT.isFixedLengthVector())
+ return SDValue();
+
+ unsigned SubBits = SubVT.getFixedSizeInBits();
+ if (SubBits >= 128 || SubBits == 64)
+ return SDValue();
+
+ EVT EltVT = SubVT.getVectorElementType();
+ unsigned EltBits = EltVT.getFixedSizeInBits();
+ if (EltBits % 8 != 0)
+ return SDValue();
+
+ // ReconstructShuffle's tbl3/tbl4 path requires more than 4 elements.
+ unsigned SubElts = SubVT.getVectorNumElements();
+ unsigned WideElts = SubElts * Factor;
+ if (WideElts <= 4)
+ return SDValue();
+
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideElts);
+ assert(WideVT == ST->getMemoryVT() &&
+ "Gathered vector must match the interleaved store's memory type");
+
+ SDLoc DL(N);
+ EVT ExtractVT =
+ EltVT.isInteger() && EltVT.bitsLT(MVT::i32) ? MVT::i32 : EltVT;
+ SmallVector<SDValue, 16> Elts(WideElts);
+ for (unsigned I = 0; I < WideElts; ++I)
+ Elts[I] =
+ DAG.getExtractVectorElt(DL, ExtractVT, Inputs[I % Factor], I / Factor);
+ SDValue Wide = DAG.getBuildVector(WideVT, DL, Elts);
+
+ return DAG.getStore(ST->getChain(), DL, Wide, ST->getBasePtr(),
+ ST->getMemOperand());
+}
+
static SDValue
performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG) {
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index 2e1578bb320fe..f3891af2286e3 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -1321,10 +1321,8 @@ define void @loop3_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr n
; CHECK-LABEL: loop3_intrinsic:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: subs w8, w2, #1
-; CHECK-NEXT: b.lt .LBB5_7
+; CHECK-NEXT: b.lt .LBB5_6
; CHECK-NEXT: // %bb.1: // %for.body.preheader
-; CHECK-NEXT: sub sp, sp, #32
-; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: cmp w8, #2
; CHECK-NEXT: b.ls .LBB5_3
; CHECK-NEXT: // %bb.2: // %vector.memcheck
@@ -1334,7 +1332,7 @@ define void @loop3_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr n
; CHECK-NEXT: add x9, x1, x9, lsl #2
; CHECK-NEXT: cmp x10, x1
; CHECK-NEXT: ccmp x9, x0, #0, hi
-; CHECK-NEXT: b.ls .LBB5_8
+; CHECK-NEXT: b.ls .LBB5_7
; CHECK-NEXT: .LBB5_3:
; CHECK-NEXT: mov w10, wzr
; CHECK-NEXT: mov x8, x1
@@ -1370,11 +1368,9 @@ define void @loop3_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr n
; CHECK-NEXT: stur b3, [x9, #2]
; CHECK-NEXT: add x9, x9, #3
; CHECK-NEXT: b.ne .LBB5_5
-; CHECK-NEXT: .LBB5_6:
-; CHECK-NEXT: add sp, sp, #32
-; CHECK-NEXT: .LBB5_7: // %for.cond.cleanup
+; CHECK-NEXT: .LBB5_6: // %for.cond.cleanup
; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB5_8: // %vector.ph
+; CHECK-NEXT: .LBB5_7: // %vector.ph
; CHECK-NEXT: add x11, x8, #1
; CHECK-NEXT: mov w8, #1132396544 // =0x437f0000
; CHECK-NEXT: adrp x12, .LCPI5_0
@@ -1382,14 +1378,13 @@ define void @loop3_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr n
; CHECK-NEXT: dup v0.4s, w8
; CHECK-NEXT: ldr q1, [x12, :lo12:.LCPI5_0]
; CHECK-NEXT: add x9, x10, x10, lsl #1
-; CHECK-NEXT: add x12, sp, #8
-; CHECK-NEXT: and x13, x11, #0x1fffffffc
+; CHECK-NEXT: and x12, x11, #0x1fffffffc
; CHECK-NEXT: add x8, x1, x9, lsl #2
; CHECK-NEXT: add x9, x0, x9
-; CHECK-NEXT: .LBB5_9: // %vector.body
+; CHECK-NEXT: .LBB5_8: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ld3 { v2.4s, v3.4s, v4.4s }, [x1], #48
-; CHECK-NEXT: subs x13, x13, #4
+; CHECK-NEXT: subs x12, x12, #4
; CHECK-NEXT: fcmgt v5.4s, v2.4s, v0.4s
; CHECK-NEXT: fcmgt v6.4s, v3.4s, v0.4s
; CHECK-NEXT: fcmgt v7.4s, v4.4s, v0.4s
@@ -1408,16 +1403,13 @@ define void @loop3_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr n
; CHECK-NEXT: xtn v5.4h, v3.4s
; CHECK-NEXT: xtn v6.4h, v4.4s
; CHECK-NEXT: xtn v7.4h, v2.4s
-; CHECK-NEXT: st3 { v5.4h, v6.4h, v7.4h }, [x12]
-; CHECK-NEXT: ldp d3, d4, [sp, #16]
-; CHECK-NEXT: ldr d2, [sp, #8]
-; CHECK-NEXT: tbl v2.16b, { v2.16b, v3.16b, v4.16b }, v1.16b
+; CHECK-NEXT: tbl v2.16b, { v5.16b, v6.16b, v7.16b }, v1.16b
; CHECK-NEXT: mov s3, v2.s[2]
; CHECK-NEXT: str d2, [x0]
; CHECK-NEXT: str s3, [x0, #8]
; CHECK-NEXT: add x0, x0, #12
-; CHECK-NEXT: b.ne .LBB5_9
-; CHECK-NEXT: // %bb.10: // %middle.block
+; CHECK-NEXT: b.ne .LBB5_8
+; CHECK-NEXT: // %bb.9: // %middle.block
; CHECK-NEXT: cmp x11, x10
; CHECK-NEXT: b.ne .LBB5_4
; CHECK-NEXT: b .LBB5_6
@@ -1596,47 +1588,43 @@ define void @loop4_intrinsic(ptr noalias nocapture noundef writeonly %dst, ptr n
; CHECK-NEXT: .LBB6_7: // %vector.ph
; CHECK-NEXT: add x11, x8, #1
; CHECK-NEXT: mov w8, #1132396544 // =0x437f0000
+; CHECK-NEXT: adrp x12, .LCPI6_0
; CHECK-NEXT: and x10, x11, #0x1fffffffc
; CHECK-NEXT: dup v0.4s, w8
-; CHECK-NEXT: and x12, x11, #0x1fffffffc
+; CHECK-NEXT: ldr q1, [x12, :lo12:.LCPI6_0]
; CHECK-NEXT: add x8, x1, x10, lsl #4
; CHECK-NEXT: add x9, x0, x10, lsl #2
+; CHECK-NEXT: and x12, x11, #0x1fffffffc
; CHECK-NEXT: .LBB6_8: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ld4 { v1.4s, v2.4s, v3.4s, v4.4s }, [x1], #64
+; CHECK-NEXT: ld4 { v2.4s, v3.4s, v4.4s, v5.4s }, [x1], #64
; CHECK-NEXT: subs x12, x12, #4
-; CHECK-NEXT: fcmgt v5.4s, v2.4s, v0.4s
-; CHECK-NEXT: fcmgt v6.4s, v4.4s, v0.4s
-; CHECK-NEXT: fcmgt v7.4s, v1.4s, v0.4s
-; CHECK-NEXT: fcmgt v16.4s, v3.4s, v0.4s
-; CHECK-NEXT: fcmlt v17.4s, v1.4s, #0.0
+; CHECK-NEXT: fcmgt v6.4s, v2.4s, v0.4s
+; CHECK-NEXT: fcmgt v7.4s, v3.4s, v0.4s
+; CHECK-NEXT: fcmgt v16.4s, v4.4s, v0.4s
+; CHECK-NEXT: fcmgt v17.4s, v5.4s, v0.4s
; CHECK-NEXT: fcmlt v18.4s, v2.4s, #0.0
-; CHECK-NEXT: fcmlt v19.4s, v4.4s, #0.0
-; CHECK-NEXT: bsl v5.16b, v0.16b, v2.16b
-; CHECK-NEXT: bsl v6.16b, v0.16b, v4.16b
-; CHECK-NEXT: bsl v7.16b, v0.16b, v1.16b
-; CHECK-NEXT: bsl v16.16b, v0.16b, v3.16b
-; CHECK-NEXT: fcmlt v1.4s, v3.4s, #0.0
-; CHECK-NEXT: bic v2.16b, v5.16b, v18.16b
-; CHECK-NEXT: bic v3.16b, v6.16b, v19.16b
-; CHECK-NEXT: bic v4.16b, v7.16b, v17.16b
-; CHECK-NEXT: bic v1.16b, v16.16b, v1.16b
-; CHECK-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-NEXT: fcmlt v19.4s, v3.4s, #0.0
+; CHECK-NEXT: fcmlt v20.4s, v4.4s, #0.0
+; CHECK-NEXT: bsl v6.16b, v0.16b, v2.16b
+; CHECK-NEXT: bsl v7.16b, v0.16b, v3.16b
+; CHECK-NEXT: bsl v16.16b, v0.16b, v4.16b
+; CHECK-NEXT: bsl v17.16b, v0.16b, v5.16b
+; CHECK-NEXT: fcmlt v2.4s, v5.4s, #0.0
+; CHECK-NEXT: bic v3.16b, v6.16b, v18.16b
+; CHECK-NEXT: bic v4.16b, v7.16b, v19.16b
+; CHECK-NEXT: bic v5.16b, v16.16b, v20.16b
+; CHECK-NEXT: bic v2.16b, v17.16b, v2.16b
; CHECK-NEXT: fcvtzs v3.4s, v3.4s
; CHECK-NEXT: fcvtzs v4.4s, v4.4s
-; CHECK-NEXT: fcvtzs v1.4s, v1.4s
-; CHECK-NEXT: xtn v2.4h, v2.4s
-; CHECK-NEXT: xtn v3.4h, v3.4s
-; CHECK-NEXT: xtn v4.4h, v4.4s
-; CHECK-NEXT: xtn v1.4h, v1.4s
-; CHECK-NEXT: zip2 v5.4h, v2.4h, v3.4h
-; CHECK-NEXT: zip1 v2.4h, v2.4h, v3.4h
-; CHECK-NEXT: zip2 v6.4h, v4.4h, v1.4h
-; CHECK-NEXT: zip1 v1.4h, v4.4h, v1.4h
-; CHECK-NEXT: zip1 v3.8h, v6.8h, v5.8h
-; CHECK-NEXT: zip1 v1.8h, v1.8h, v2.8h
-; CHECK-NEXT: uzp1 v1.16b, v1.16b, v3.16b
-; CHECK-NEXT: str q1, [x0], #16
+; CHECK-NEXT: fcvtzs v5.4s, v5.4s
+; CHECK-NEXT: fcvtzs v2.4s, v2.4s
+; CHECK-NEXT: xtn v16.4h, v3.4s
+; CHECK-NEXT: xtn v17.4h, v4.4s
+; CHECK-NEXT: xtn v18.4h, v5.4s
+; CHECK-NEXT: xtn v19.4h, v2.4s
+; CHECK-NEXT: tbl v2.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-NEXT: str q2, [x0], #16
; CHECK-NEXT: b.ne .LBB6_8
; CHECK-NEXT: // %bb.9: // %middle.block
; CHECK-NEXT: cmp x11, x10
More information about the llvm-commits
mailing list