[llvm] [AArch64] Add combine for interleave deinterleave (PR #208414)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 07:22:22 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/208414
>From ac6b491b7872aae612b80be4cca5d67cdf452b44 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Fri, 3 Jul 2026 17:42:09 +0000
Subject: [PATCH 1/4] [CodeGen] Expand unsupported (de)interleave intrinsics
Not all targets support lowering of (de)interleave natively.
Add a generic fallback that lowers them to shuffle, allowing
targets to reuse existing lowering paths.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 6 +
.../SelectionDAG/SelectionDAGBuilder.cpp | 54 +++--
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 5 +
llvm/lib/Target/RISCV/RISCVISelLowering.h | 4 +
.../AArch64/fixed-vector-deinterleave.ll | 111 +++++++++-
.../AArch64/fixed-vector-interleave.ll | 62 ++++++
.../rvv/fixed-vectors-deinterleave-load.ll | 20 +-
.../RISCV/rvv/vector-deinterleave-fixed.ll | 197 ++++++------------
8 files changed, 285 insertions(+), 174 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index f7f14fed05393..d171fc0a26a13 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3306,6 +3306,12 @@ class LLVM_ABI TargetLoweringBase {
/// Default to be the minimum interleave factor: 2.
virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; }
+ /// Return true if target supports interleave intrinsics.
+ virtual bool isInterleaveIntrinsicSupported(unsigned /*Factor*/,
+ EVT VT) const {
+ return VT.isScalableVector();
+ }
+
/// Lower an interleaved load to target specific intrinsics. Return
/// true on success.
///
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 27a6cf37807b5..43542dd3a371d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -13029,34 +13029,44 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I,
unsigned Factor) {
auto DL = getCurSDLoc();
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue InVec = getValue(I.getOperand(0));
+ EVT InVT = InVec.getValueType();
SmallVector<EVT, 4> ValueVTs;
- ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
- ValueVTs);
+ ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
EVT OutVT = ValueVTs[0];
unsigned OutNumElts = OutVT.getVectorMinNumElements();
- SmallVector<SDValue, 4> SubVecs(Factor);
- for (unsigned i = 0; i != Factor; ++i) {
- assert(ValueVTs[i] == OutVT && "Expected VTs to be the same");
- SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
- DAG.getVectorIdxConstant(OutNumElts * i, DL));
- }
-
- // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
- // from existing legalisation and combines.
- if (OutVT.isFixedLengthVector() && Factor == 2) {
- SDValue Even = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
- createStrideMask(0, 2, OutNumElts));
- SDValue Odd = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1],
- createStrideMask(1, 2, OutNumElts));
- SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
- setValue(&I, Res);
+ // Use VECTOR_SHUFFLE for fixed-length vectors that the target does not lower
+ // natively.
+ if (!TLI.isInterleaveIntrinsicSupported(Factor, InVT)) {
+ unsigned WideNumElts = InVT.getVectorNumElements();
+
+ SmallVector<SDValue, 8> Results;
+ Results.reserve(Factor);
+ for (unsigned I = 0; I != Factor; ++I) {
+ SmallVector<int, 16> Mask(WideNumElts, -1);
+ for (unsigned J = 0; J != OutNumElts; ++J)
+ Mask[J] = I + J * Factor;
+ SDValue Shuffle =
+ DAG.getVectorShuffle(InVT, DL, InVec, DAG.getUNDEF(InVT), Mask);
+ Results.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, Shuffle,
+ DAG.getVectorIdxConstant(0, DL)));
+ }
+
+ setValue(&I, DAG.getMergeValues(Results, DL));
return;
}
+ SmallVector<SDValue, 4> SubVecs(Factor);
+ for (unsigned I = 0; I != Factor; ++I) {
+ assert(ValueVTs[I] == OutVT && "Expected VTs to be the same");
+ SubVecs[I] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
+ DAG.getVectorIdxConstant(OutNumElts * I, DL));
+ }
+
SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
DAG.getVTList(ValueVTs), SubVecs);
setValue(&I, Res);
@@ -13076,13 +13086,13 @@ void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I,
"Expected VTs to be the same");
}
- // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
- // from existing legalisation and combines.
- if (OutVT.isFixedLengthVector() && Factor == 2) {
+ // Use VECTOR_SHUFFLE for fixed-length vectors that the target does not lower
+ // natively.
+ if (!TLI.isInterleaveIntrinsicSupported(Factor, OutVT)) {
unsigned NumElts = InVT.getVectorMinNumElements();
SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVecs);
setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
- createInterleaveMask(NumElts, 2)));
+ createInterleaveMask(NumElts, Factor)));
return;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 69013d4010122..0e86aae1acbb5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -232,6 +232,11 @@ class AArch64TargetLowering : public TargetLowering {
unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
+ bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override {
+ return VT.isScalableVector() || Factor == 3 ||
+ (Factor > 2 && Factor % 2 == 0);
+ }
+
bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
ArrayRef<ShuffleVectorInst *> Shuffles,
ArrayRef<unsigned> Indices, unsigned Factor,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 4e7912ed815dd..792b8a03b755f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -432,6 +432,10 @@ class RISCVTargetLowering : public TargetLowering {
unsigned getMaxSupportedInterleaveFactor() const override { return 8; }
+ bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override {
+ return VT.isScalableVector() || Factor > 2;
+ }
+
bool fallBackToDAGISel(const Instruction &Inst) const override;
bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
index 8c48cc609c75a..719e5dc67336e 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
@@ -5,10 +5,8 @@
define {<2 x half>, <2 x half>} @vector_deinterleave_v2f16_v4f16(<4 x half> %vec) {
; CHECK-SD-LABEL: vector_deinterleave_v2f16_v4f16:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: dup v1.2s, v0.s[1]
-; CHECK-SD-NEXT: zip1 v2.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: trn2 v1.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: uzp1 v2.4h, v0.4h, v0.4h
+; CHECK-SD-NEXT: uzp2 v1.4h, v0.4h, v0.4h
; CHECK-SD-NEXT: fmov d0, d2
; CHECK-SD-NEXT: ret
;
@@ -48,9 +46,9 @@ define {<8 x half>, <8 x half>} @vector_deinterleave_v8f16_v16f16(<16 x half> %v
define {<2 x float>, <2 x float>} @vector_deinterleave_v2f32_v4f32(<4 x float> %vec) {
; CHECK-SD-LABEL: vector_deinterleave_v2f32_v4f32:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mov d1, v0.d[1]
-; CHECK-SD-NEXT: zip1 v2.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: zip2 v1.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: uzp1 v2.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT: uzp2 v1.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-SD-NEXT: fmov d0, d2
; CHECK-SD-NEXT: ret
;
@@ -846,3 +844,102 @@ define {<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8
%retval = call {<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>} @llvm.vector.deinterleave8.v64bf16(<64 x bfloat> %ins7)
ret {<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>} %retval
}
+
+define {<4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>} @vector_deinterleave5_v20i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c, <4 x i16> %d, <4 x i16> %e) {
+; CHECK-LABEL: vector_deinterleave5_v20i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov d6, d2
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: adrp x8, .LCPI42_1
+; CHECK-NEXT: adrp x9, .LCPI42_2
+; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI42_1]
+; CHECK-NEXT: adrp x8, .LCPI42_4
+; CHECK-NEXT: ldr q7, [x8, :lo12:.LCPI42_4]
+; CHECK-NEXT: adrp x10, .LCPI42_3
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI42_2]
+; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-NEXT: adrp x8, .LCPI42_0
+; CHECK-NEXT: mov v5.d[1], v1.d[0]
+; CHECK-NEXT: mov v6.d[1], v3.d[0]
+; CHECK-NEXT: ldr q3, [x10, :lo12:.LCPI42_3]
+; CHECK-NEXT: tbl v7.16b, { v5.16b, v6.16b }, v7.16b
+; CHECK-NEXT: tbl v1.16b, { v5.16b, v6.16b }, v0.16b
+; CHECK-NEXT: tbl v2.16b, { v5.16b, v6.16b }, v2.16b
+; CHECK-NEXT: tbl v3.16b, { v5.16b, v6.16b }, v3.16b
+; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI42_0]
+; CHECK-NEXT: tbl v0.16b, { v5.16b, v6.16b }, v0.16b
+; CHECK-NEXT: mov v7.h[3], v4.h[3]
+; CHECK-NEXT: mov v1.h[3], v4.h[0]
+; CHECK-NEXT: mov v2.h[3], v4.h[1]
+; CHECK-NEXT: mov v3.h[3], v4.h[2]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NEXT: fmov d4, d7
+; CHECK-NEXT: ret
+ %ins0 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> poison, <4 x i16> %a, i64 0)
+ %ins1 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins0, <4 x i16> %b, i64 4)
+ %ins2 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins1, <4 x i16> %c, i64 8)
+ %ins3 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins2, <4 x i16> %d, i64 12)
+ %ins4 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins3, <4 x i16> %e, i64 16)
+ %retval = call {<4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>} @llvm.vector.deinterleave5.v20i16(<20 x i16> %ins4)
+ ret {<4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>} %retval
+}
+
+define {<4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>} @vector_deinterleave7_v28i8(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d, <4 x i8> %e, <4 x i8> %f, <4 x i8> %g) {
+; CHECK-LABEL: vector_deinterleave7_v28i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 v19.8b, v0.8b, v1.8b
+; CHECK-NEXT: uzp1 v18.8b, v2.8b, v3.8b
+; CHECK-NEXT: // kill: def $d6 killed $d6 def $q6
+; CHECK-NEXT: uzp1 v4.8b, v4.8b, v5.8b
+; CHECK-NEXT: xtn v5.8b, v6.8h
+; CHECK-NEXT: mov b7, v19.b[4]
+; CHECK-NEXT: mov b16, v19.b[5]
+; CHECK-NEXT: mov b17, v19.b[6]
+; CHECK-NEXT: mov b0, v19.b[0]
+; CHECK-NEXT: mov b1, v19.b[1]
+; CHECK-NEXT: mov b2, v19.b[2]
+; CHECK-NEXT: mov b3, v19.b[3]
+; CHECK-NEXT: mov v7.b[2], v18.b[3]
+; CHECK-NEXT: mov v16.b[2], v18.b[4]
+; CHECK-NEXT: mov v17.b[2], v18.b[5]
+; CHECK-NEXT: mov v0.b[2], v19.b[7]
+; CHECK-NEXT: mov v1.b[2], v18.b[0]
+; CHECK-NEXT: mov v2.b[2], v18.b[1]
+; CHECK-NEXT: mov v3.b[2], v18.b[2]
+; CHECK-NEXT: mov v7.b[4], v4.b[2]
+; CHECK-NEXT: mov v16.b[4], v4.b[3]
+; CHECK-NEXT: mov v17.b[4], v4.b[4]
+; CHECK-NEXT: mov v0.b[4], v18.b[6]
+; CHECK-NEXT: mov v1.b[4], v18.b[7]
+; CHECK-NEXT: mov v2.b[4], v4.b[0]
+; CHECK-NEXT: mov v3.b[4], v4.b[1]
+; CHECK-NEXT: mov v7.b[6], v5.b[1]
+; CHECK-NEXT: mov v16.b[6], v5.b[2]
+; CHECK-NEXT: mov v17.b[6], v5.b[3]
+; CHECK-NEXT: mov v0.b[6], v4.b[5]
+; CHECK-NEXT: mov v1.b[6], v4.b[6]
+; CHECK-NEXT: mov v2.b[6], v4.b[7]
+; CHECK-NEXT: mov v3.b[6], v5.b[0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT: // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NEXT: fmov d4, d7
+; CHECK-NEXT: fmov d5, d16
+; CHECK-NEXT: fmov d6, d17
+; CHECK-NEXT: ret
+ %ins0 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> poison, <4 x i8> %a, i64 0)
+ %ins1 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins0, <4 x i8> %b, i64 4)
+ %ins2 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins1, <4 x i8> %c, i64 8)
+ %ins3 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins2, <4 x i8> %d, i64 12)
+ %ins4 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins3, <4 x i8> %e, i64 16)
+ %ins5 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins4, <4 x i8> %f, i64 20)
+ %ins6 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins5, <4 x i8> %g, i64 24)
+ %retval = call {<4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>} @llvm.vector.deinterleave7.v28i8(<28 x i8> %ins6)
+ ret {<4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>} %retval
+}
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
index 2d368c0ec456f..2e4f3757c9089 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
@@ -813,3 +813,65 @@ define <32 x i16> @interleave8_v32i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16
%retval = call <32 x i16> @llvm.vector.interleave8.v32i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3, <4 x i16> %vec4, <4 x i16> %vec5, <4 x i16> %vec6, <4 x i16> %vec7)
ret <32 x i16> %retval
}
+
+define <20 x i16> @interleave5_v20i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3, <4 x i16> %vec4) {
+; CHECK-LABEL: interleave5_v20i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d4 killed $d4 killed $q3_q4 def $q3_q4
+; CHECK-NEXT: fmov d5, d3
+; CHECK-NEXT: fmov d7, d2
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: adrp x9, .LCPI45_0
+; CHECK-NEXT: fmov d6, d0
+; CHECK-NEXT: adrp x10, .LCPI45_2
+; CHECK-NEXT: ldr q0, [x9, :lo12:.LCPI45_0]
+; CHECK-NEXT: adrp x9, .LCPI45_3
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI45_3]
+; CHECK-NEXT: adrp x9, .LCPI45_1
+; CHECK-NEXT: mov v7.d[1], v5.d[0]
+; CHECK-NEXT: mov v6.d[1], v1.d[0]
+; CHECK-NEXT: ldr q1, [x10, :lo12:.LCPI45_2]
+; CHECK-NEXT: tbl v3.16b, { v6.16b, v7.16b }, v0.16b
+; CHECK-NEXT: tbl v0.16b, { v6.16b, v7.16b }, v1.16b
+; CHECK-NEXT: tbl v1.16b, { v6.16b, v7.16b }, v2.16b
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI45_1]
+; CHECK-NEXT: mov v0.h[3], v4.h[3]
+; CHECK-NEXT: mov v1.h[4], v4.h[0]
+; CHECK-NEXT: tbl v2.16b, { v3.16b, v4.16b }, v2.16b
+; CHECK-NEXT: str d0, [x8, #32]
+; CHECK-NEXT: stp q1, q2, [x8]
+; CHECK-NEXT: ret
+ %retval = call <20 x i16> @llvm.vector.interleave5.v20i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3, <4 x i16> %vec4)
+ ret <20 x i16> %retval
+}
+
+define <28 x i8> @interleave7_v28i8(<4 x i8> %vec0, <4 x i8> %vec1, <4 x i8> %vec2, <4 x i8> %vec3, <4 x i8> %vec4, <4 x i8> %vec5, <4 x i8> %vec6) {
+; CHECK-LABEL: interleave7_v28i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d6 killed $d6 killed $q4_q5_q6 def $q4_q5_q6
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: adrp x9, .LCPI46_0
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: // kill: def $d5 killed $d5 killed $q4_q5_q6 def $q4_q5_q6
+; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI46_0]
+; CHECK-NEXT: // kill: def $d4 killed $d4 killed $q4_q5_q6 def $q4_q5_q6
+; CHECK-NEXT: adrp x9, .LCPI46_1
+; CHECK-NEXT: tbl v4.16b, { v4.16b, v5.16b, v6.16b }, v1.16b
+; CHECK-NEXT: uzp1 v3.16b, v0.16b, v2.16b
+; CHECK-NEXT: ldr q0, [x9, :lo12:.LCPI46_1]
+; CHECK-NEXT: adrp x9, .LCPI46_2
+; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI46_2]
+; CHECK-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b
+; CHECK-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v1.16b
+; CHECK-NEXT: mov s2, v0.s[2]
+; CHECK-NEXT: str q1, [x8]
+; CHECK-NEXT: str d0, [x8, #16]
+; CHECK-NEXT: str s2, [x8, #24]
+; CHECK-NEXT: ret
+ %retval = call <28 x i8> @llvm.vector.interleave7.v28i8(<4 x i8> %vec0, <4 x i8> %vec1, <4 x i8> %vec2, <4 x i8> %vec3, <4 x i8> %vec4, <4 x i8> %vec5, <4 x i8> %vec6)
+ ret <28 x i8> %retval
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
index ffc9067a78604..d81ad8206ba6c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
@@ -12,24 +12,16 @@ define {<16 x i1>, <16 x i1>} @vector_deinterleave_load_v16i1_v32i1(ptr %p) {
; CHECK-NEXT: li a1, 32
; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
; CHECK-NEXT: vlm.v v0, (a0)
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmv.v.i v8, 0
-; CHECK-NEXT: vmerge.vim v9, v8, 1, v0
-; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, ma
-; CHECK-NEXT: vslidedown.vi v0, v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vnsrl.wi v10, v9, 0
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vnsrl.wi v9, v9, 8
-; CHECK-NEXT: vnsrl.wi v11, v8, 0
-; CHECK-NEXT: vnsrl.wi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vslideup.vi v10, v11, 8
-; CHECK-NEXT: vslideup.vi v9, v8, 8
+; CHECK-NEXT: vnsrl.wi v10, v8, 0
+; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
; CHECK-NEXT: vmsne.vi v0, v10, 0
-; CHECK-NEXT: vmsne.vi v8, v9, 0
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vnsrl.wi v10, v8, 8
+; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT: vmsne.vi v8, v10, 0
; CHECK-NEXT: ret
%vec = load <32 x i1>, ptr %p
%deinterleaved.results = call {<16 x i1>, <16 x i1>} @llvm.vector.deinterleave2.v32i1(<32 x i1> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index bce071e988f0e..f002f4897bd66 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -8,24 +8,18 @@
define {<16 x i1>, <16 x i1>} @vector_deinterleave_v16i1_v32i1(<32 x i1> %vec) {
; CHECK-LABEL: vector_deinterleave_v16i1_v32i1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: li a0, 32
+; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma
; CHECK-NEXT: vmv.v.i v8, 0
-; CHECK-NEXT: vmerge.vim v9, v8, 1, v0
-; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, ma
-; CHECK-NEXT: vslidedown.vi v0, v0, 2
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vnsrl.wi v10, v9, 0
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT: vnsrl.wi v9, v9, 8
-; CHECK-NEXT: vnsrl.wi v11, v8, 0
-; CHECK-NEXT: vnsrl.wi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vslideup.vi v10, v11, 8
-; CHECK-NEXT: vslideup.vi v9, v8, 8
+; CHECK-NEXT: vnsrl.wi v10, v8, 0
+; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma
; CHECK-NEXT: vmsne.vi v0, v10, 0
-; CHECK-NEXT: vmsne.vi v8, v9, 0
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vnsrl.wi v10, v8, 8
+; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT: vmsne.vi v8, v10, 0
; CHECK-NEXT: ret
%retval = call {<16 x i1>, <16 x i1>} @llvm.vector.deinterleave2.v32i1(<32 x i1> %vec)
ret {<16 x i1>, <16 x i1>} %retval
@@ -74,22 +68,22 @@ ret {<4 x i32>, <4 x i32>} %retval
define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) {
; V-LABEL: vector_deinterleave_v2i64_v4i64:
; V: # %bb.0:
-; V-NEXT: vsetivli zero, 2, e64, m2, ta, ma
-; V-NEXT: vslidedown.vi v10, v8, 2
-; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
-; V-NEXT: vmv.v.i v0, 1
+; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
+; V-NEXT: vslidedown.vi v10, v8, 1
+; V-NEXT: vslidedown.vi v10, v8, 2, v0.t
+; V-NEXT: vslidedown.vi v8, v8, 1, v0.t
; V-NEXT: vmv1r.v v9, v10
-; V-NEXT: vslidedown.vi v9, v8, 1, v0.t
-; V-NEXT: vslideup.vi v8, v10, 1
; V-NEXT: ret
;
; ZVZIP-LABEL: vector_deinterleave_v2i64_v4i64:
; ZVZIP: # %bb.0:
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vunzipe.v v10, v8
-; ZVZIP-NEXT: vunzipo.v v11, v8
-; ZVZIP-NEXT: vmv.v.v v8, v10
-; ZVZIP-NEXT: vmv.v.v v9, v11
+; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma
+; ZVZIP-NEXT: vunzipe.v v12, v8
+; ZVZIP-NEXT: vunzipo.v v14, v8
+; ZVZIP-NEXT: vmv1r.v v8, v12
+; ZVZIP-NEXT: vmv1r.v v9, v14
; ZVZIP-NEXT: ret
%retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec)
ret {<2 x i64>, <2 x i64>} %retval
@@ -98,44 +92,24 @@ ret {<2 x i64>, <2 x i64>} %retval
define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
; V-LABEL: vector_deinterleave_v4i64_v8i64:
; V: # %bb.0:
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v0, 8
-; V-NEXT: vsetivli zero, 4, e64, m4, ta, ma
-; V-NEXT: vslidedown.vi v16, v8, 4
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; V-NEXT: vslideup.vi v12, v16, 2
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v10, 2
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
-; V-NEXT: vslideup.vi v12, v16, 1, v0.t
-; V-NEXT: vmv2r.v v14, v8
-; V-NEXT: vmv1r.v v0, v10
-; V-NEXT: vslidedown.vi v14, v8, 1, v0.t
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v11, 12
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
-; V-NEXT: vslidedown.vi v18, v8, 1
-; V-NEXT: vmv1r.v v0, v11
-; V-NEXT: vmerge.vvm v12, v14, v12, v0
-; V-NEXT: vmv1r.v v0, v10
-; V-NEXT: vslidedown.vi v18, v8, 2, v0.t
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v0, 4
-; V-NEXT: vmv2r.v v8, v16
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
-; V-NEXT: vslideup.vi v8, v16, 1, v0.t
-; V-NEXT: vmv1r.v v0, v11
-; V-NEXT: vmerge.vvm v10, v18, v8, v0
+; V-NEXT: li a0, 85
+; V-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; V-NEXT: vmv.s.x v16, a0
+; V-NEXT: vcompress.vm v12, v8, v16
+; V-NEXT: li a0, 170
+; V-NEXT: vmv.s.x v14, a0
+; V-NEXT: vcompress.vm v16, v8, v14
; V-NEXT: vmv2r.v v8, v12
+; V-NEXT: vmv2r.v v10, v16
; V-NEXT: ret
;
; ZVZIP-LABEL: vector_deinterleave_v4i64_v8i64:
; ZVZIP: # %bb.0:
-; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; ZVZIP-NEXT: vunzipe.v v12, v8
-; ZVZIP-NEXT: vunzipo.v v14, v8
-; ZVZIP-NEXT: vmv.v.v v8, v12
-; ZVZIP-NEXT: vmv.v.v v10, v14
+; ZVZIP-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; ZVZIP-NEXT: vunzipe.v v16, v8
+; ZVZIP-NEXT: vunzipo.v v20, v8
+; ZVZIP-NEXT: vmv2r.v v8, v16
+; ZVZIP-NEXT: vmv2r.v v10, v20
; ZVZIP-NEXT: ret
%retval = call {<4 x i64>, <4 x i64>} @llvm.vector.deinterleave2.v8i64(<8 x i64> %vec)
ret {<4 x i64>, <4 x i64>} %retval
@@ -144,37 +118,18 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec) {
; V-LABEL: vector_deinterleave_v8i64_v16i64:
; V: # %bb.0:
-; V-NEXT: li a0, 85
-; V-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; V-NEXT: lui a0, 5
+; V-NEXT: addi a0, a0, 1365
+; V-NEXT: vsetivli zero, 16, e64, m8, ta, ma
+; V-NEXT: vmv.s.x v24, a0
+; V-NEXT: vcompress.vm v16, v8, v24
+; V-NEXT: lui a0, 11
+; V-NEXT: addi a0, a0, -1366
; V-NEXT: vmv.s.x v20, a0
-; V-NEXT: vcompress.vm v16, v8, v20
-; V-NEXT: vsetivli zero, 8, e64, m8, ta, ma
-; V-NEXT: vslidedown.vi v24, v8, 8
-; V-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; V-NEXT: vid.v v12
-; V-NEXT: vadd.vv v20, v12, v12
-; V-NEXT: vmv.v.i v0, -16
-; V-NEXT: vadd.vi v12, v20, -8
-; V-NEXT: vsetvli zero, zero, e64, m4, ta, mu
-; V-NEXT: vrgatherei16.vv v16, v24, v12, v0.t
-; V-NEXT: li a0, 170
-; V-NEXT: vmv.s.x v21, a0
-; V-NEXT: vcompress.vm v12, v8, v21
-; V-NEXT: vsetvli zero, zero, e16, m1, ta, ma
-; V-NEXT: vadd.vi v8, v20, -7
-; V-NEXT: vsetvli zero, zero, e64, m4, ta, mu
-; V-NEXT: vrgatherei16.vv v12, v24, v8, v0.t
-; V-NEXT: vmv.v.v v8, v16
+; V-NEXT: vcompress.vm v24, v8, v20
+; V-NEXT: vmv4r.v v8, v16
+; V-NEXT: vmv4r.v v12, v24
; V-NEXT: ret
-;
-; ZVZIP-LABEL: vector_deinterleave_v8i64_v16i64:
-; ZVZIP: # %bb.0:
-; ZVZIP-NEXT: vsetivli zero, 8, e64, m4, ta, ma
-; ZVZIP-NEXT: vunzipe.v v16, v8
-; ZVZIP-NEXT: vunzipo.v v20, v8
-; ZVZIP-NEXT: vmv.v.v v8, v16
-; ZVZIP-NEXT: vmv.v.v v12, v20
-; ZVZIP-NEXT: ret
%retval = call {<8 x i64>, <8 x i64>} @llvm.vector.deinterleave2.v16i64(<16 x i64> %vec)
ret {<8 x i64>, <8 x i64>} %retval
}
@@ -800,22 +755,22 @@ ret {<4 x float>, <4 x float>} %retval
define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double> %vec) {
; V-LABEL: vector_deinterleave_v2f64_v4f64:
; V: # %bb.0:
-; V-NEXT: vsetivli zero, 2, e64, m2, ta, ma
-; V-NEXT: vslidedown.vi v10, v8, 2
-; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu
-; V-NEXT: vmv.v.i v0, 1
+; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
+; V-NEXT: vmv.v.i v0, 2
+; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
+; V-NEXT: vslidedown.vi v10, v8, 1
+; V-NEXT: vslidedown.vi v10, v8, 2, v0.t
+; V-NEXT: vslidedown.vi v8, v8, 1, v0.t
; V-NEXT: vmv1r.v v9, v10
-; V-NEXT: vslidedown.vi v9, v8, 1, v0.t
-; V-NEXT: vslideup.vi v8, v10, 1
; V-NEXT: ret
;
; ZVZIP-LABEL: vector_deinterleave_v2f64_v4f64:
; ZVZIP: # %bb.0:
-; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; ZVZIP-NEXT: vunzipe.v v10, v8
-; ZVZIP-NEXT: vunzipo.v v11, v8
-; ZVZIP-NEXT: vmv.v.v v8, v10
-; ZVZIP-NEXT: vmv.v.v v9, v11
+; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma
+; ZVZIP-NEXT: vunzipe.v v12, v8
+; ZVZIP-NEXT: vunzipo.v v14, v8
+; ZVZIP-NEXT: vmv1r.v v8, v12
+; ZVZIP-NEXT: vmv1r.v v9, v14
; ZVZIP-NEXT: ret
%retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec)
ret {<2 x double>, <2 x double>} %retval
@@ -824,44 +779,24 @@ ret {<2 x double>, <2 x double>} %retval
define {<4 x double>, <4 x double>} @vector_deinterleave_v4f64_v8f64(<8 x double> %vec) {
; V-LABEL: vector_deinterleave_v4f64_v8f64:
; V: # %bb.0:
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v0, 8
-; V-NEXT: vsetivli zero, 4, e64, m4, ta, ma
-; V-NEXT: vslidedown.vi v16, v8, 4
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; V-NEXT: vslideup.vi v12, v16, 2
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v10, 2
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
-; V-NEXT: vslideup.vi v12, v16, 1, v0.t
-; V-NEXT: vmv2r.v v14, v8
-; V-NEXT: vmv1r.v v0, v10
-; V-NEXT: vslidedown.vi v14, v8, 1, v0.t
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v11, 12
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
-; V-NEXT: vslidedown.vi v18, v8, 1
-; V-NEXT: vmv1r.v v0, v11
-; V-NEXT: vmerge.vvm v12, v14, v12, v0
-; V-NEXT: vmv1r.v v0, v10
-; V-NEXT: vslidedown.vi v18, v8, 2, v0.t
-; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
-; V-NEXT: vmv.v.i v0, 4
-; V-NEXT: vmv2r.v v8, v16
-; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu
-; V-NEXT: vslideup.vi v8, v16, 1, v0.t
-; V-NEXT: vmv1r.v v0, v11
-; V-NEXT: vmerge.vvm v10, v18, v8, v0
+; V-NEXT: li a0, 85
+; V-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; V-NEXT: vmv.s.x v16, a0
+; V-NEXT: vcompress.vm v12, v8, v16
+; V-NEXT: li a0, 170
+; V-NEXT: vmv.s.x v14, a0
+; V-NEXT: vcompress.vm v16, v8, v14
; V-NEXT: vmv2r.v v8, v12
+; V-NEXT: vmv2r.v v10, v16
; V-NEXT: ret
;
; ZVZIP-LABEL: vector_deinterleave_v4f64_v8f64:
; ZVZIP: # %bb.0:
-; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma
-; ZVZIP-NEXT: vunzipe.v v12, v8
-; ZVZIP-NEXT: vunzipo.v v14, v8
-; ZVZIP-NEXT: vmv.v.v v8, v12
-; ZVZIP-NEXT: vmv.v.v v10, v14
+; ZVZIP-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; ZVZIP-NEXT: vunzipe.v v16, v8
+; ZVZIP-NEXT: vunzipo.v v20, v8
+; ZVZIP-NEXT: vmv2r.v v8, v16
+; ZVZIP-NEXT: vmv2r.v v10, v20
; ZVZIP-NEXT: ret
%retval = call {<4 x double>, <4 x double>} @llvm.vector.deinterleave2.v8f64(<8 x double> %vec)
ret {<4 x double>, <4 x double>} %retval
>From a6e94f119504f3cc8bfa9775a34c3fafa90ac8e7 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Tue, 14 Jul 2026 13:35:40 +0000
Subject: [PATCH 2/4] [CodeGen] Add Widening support for vector_interleave
---
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 40 ++++++++++++
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 3 +-
.../complex-deinterleaving-reductions.ll | 50 ++++++++-------
.../AArch64/complex-deinterleaving-splat.ll | 45 +++++++------
.../AArch64/fixed-vector-deinterleave.ll | 63 ++++++++++---------
.../AArch64/fixed-vector-interleave.ll | 60 ++++++------------
.../fixed_masked_deinterleaved_loads.ll | 28 ++++-----
.../fixed_masked_interleaved_stores.ll | 24 +++----
9 files changed, 170 insertions(+), 144 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index d4d56a9563f71..5a7df7c1b73c6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -1079,6 +1079,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N);
SDValue WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N);
void WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N);
+ void WidenVecRes_VECTOR_INTERLEAVE(SDNode *N);
SDValue WidenVecRes_Ternary(SDNode *N);
SDValue WidenVecRes_Binary(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 3ac1ca367abc8..4742894ff59b0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -5331,6 +5331,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::VECTOR_DEINTERLEAVE:
WidenVecRes_VECTOR_DEINTERLEAVE(N);
break;
+ case ISD::VECTOR_INTERLEAVE:
+ WidenVecRes_VECTOR_INTERLEAVE(N);
+ break;
case ISD::ADD: case ISD::VP_ADD:
case ISD::AND: case ISD::VP_AND:
@@ -7523,6 +7526,43 @@ void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
SetWidenedVector(SDValue(N, Idx), NewRes.getValue(Idx));
}
+void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT EltVT = VT.getVectorElementType();
+ ElementCount OrigEC = VT.getVectorElementCount();
+ unsigned Factor = N->getNumOperands();
+ SDLoc DL(N);
+
+ EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ ElementCount WidenEC = WidenVT.getVectorElementCount();
+
+ SmallVector<SDValue, 8> WidenOps(Factor);
+ for (unsigned Idx = 0U; Idx < Factor; ++Idx)
+ WidenOps[Idx] = GetWidenedVector(N->getOperand(Idx));
+
+ SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
+ SDValue Interleaved =
+ DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, WidenVTs, WidenOps);
+
+ EVT PackedWidenVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
+ WidenEC.multiplyCoefficientBy(Factor));
+ SmallVector<SDValue, 8> Slices(Factor);
+ for (unsigned Idx = 0; Idx != Factor; ++Idx)
+ Slices[Idx] = Interleaved.getValue(Idx);
+
+ SDValue Packed =
+ DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedWidenVT, Slices);
+
+ for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
+ SDValue Narrow = DAG.getExtractSubvector(
+ DL, VT, Packed,
+ OrigEC.multiplyCoefficientBy(Idx).getKnownMinValue());
+ SDValue Wide =
+ DAG.getInsertSubvector(DL, DAG.getUNDEF(WidenVT), Narrow, /*Idx=*/0U);
+ SetWidenedVector(SDValue(N, Idx), Wide);
+ }
+}
+
SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
assert(N->getValueType(0).isVector() &&
N->getOperand(0).getValueType().isVector() &&
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 0e86aae1acbb5..b67aa4c966181 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -233,8 +233,7 @@ class AArch64TargetLowering : public TargetLowering {
unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override {
- return VT.isScalableVector() || Factor == 3 ||
- (Factor > 2 && Factor % 2 == 0);
+ return VT.isScalableVector() || Factor == 3 || Factor % 2 == 0;
}
bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
index 355adec955e4b..3e3e3389c146d 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll
@@ -31,8 +31,8 @@ define dso_local %"struct.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90
; CHECK-NEXT: b.ne .LBB0_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: faddp d1, v2.2d
; CHECK-NEXT: ret
@@ -80,10 +80,13 @@ middle.block: ; preds = %vector.body
define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
; CHECK-LABEL: complex_mul_nonzero_init_v2f64:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: adrp x8, .LCPI1_0
+; CHECK-NEXT: adrp x9, .LCPI1_1
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI1_1]
; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: zip2 v0.2d, v2.2d, v1.2d
+; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d
; CHECK-NEXT: .LBB1_1: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: add x9, x0, x8
@@ -98,8 +101,8 @@ define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90
; CHECK-NEXT: b.ne .LBB1_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: faddp d1, v2.2d
; CHECK-NEXT: ret
@@ -143,14 +146,17 @@ middle.block: ; preds = %vector.body
define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-LABEL: complex_mul_v2f64_unrolled:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: adrp x9, .LCPI2_1
; CHECK-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI2_1]
; CHECK-NEXT: add x8, x0, #32
; CHECK-NEXT: add x9, x1, #32
; CHECK-NEXT: mov x10, #-100 // =0xffffffffffffff9c
+; CHECK-NEXT: zip2 v0.2d, v2.2d, v1.2d
+; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: .LBB2_1: // %vector.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldp q5, q4, [x8, #-32]
@@ -158,24 +164,24 @@ define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-NEXT: ldp q7, q6, [x9, #-32]
; CHECK-NEXT: ldp q17, q16, [x8], #64
; CHECK-NEXT: ldp q19, q18, [x9], #64
-; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #0
-; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #0
-; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #0
-; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #0
-; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #90
-; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #90
-; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #90
-; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #90
+; CHECK-NEXT: fcmla v1.2d, v7.2d, v5.2d, #0
+; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #0
+; CHECK-NEXT: fcmla v3.2d, v19.2d, v17.2d, #0
+; CHECK-NEXT: fcmla v2.2d, v18.2d, v16.2d, #0
+; CHECK-NEXT: fcmla v1.2d, v7.2d, v5.2d, #90
+; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #90
+; CHECK-NEXT: fcmla v3.2d, v19.2d, v17.2d, #90
+; CHECK-NEXT: fcmla v2.2d, v18.2d, v16.2d, #90
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %middle.block
-; CHECK-NEXT: zip2 v4.2d, v0.2d, v3.2d
-; CHECK-NEXT: zip1 v0.2d, v0.2d, v3.2d
-; CHECK-NEXT: zip2 v3.2d, v2.2d, v1.2d
-; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d
-; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: uzp2 v4.2d, v3.2d, v2.2d
+; CHECK-NEXT: uzp1 v2.2d, v3.2d, v2.2d
+; CHECK-NEXT: uzp2 v3.2d, v1.2d, v0.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d
; CHECK-NEXT: fadd v1.2d, v4.2d, v3.2d
-; CHECK-NEXT: faddp d0, v0.2d
+; CHECK-NEXT: fadd v0.2d, v2.2d, v0.2d
; CHECK-NEXT: faddp d1, v1.2d
+; CHECK-NEXT: faddp d0, v0.2d
; CHECK-NEXT: ret
entry:
%scevgep = getelementptr i8, ptr %a, i64 32
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll
index ad9240b0922bd..f8c7a25f47185 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll
@@ -9,21 +9,23 @@ target triple = "aarch64"
define <4 x double> @complex_mul_const(<4 x double> %a, <4 x double> %b) {
; CHECK-LABEL: complex_mul_const:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v6.2d, #0000000000000000
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: adrp x8, .LCPI0_0
; CHECK-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NEXT: fcmla v6.2d, v1.2d, v3.2d, #0
-; CHECK-NEXT: fcmla v5.2d, v0.2d, v2.2d, #0
-; CHECK-NEXT: fcmla v6.2d, v1.2d, v3.2d, #90
-; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: fcmla v5.2d, v0.2d, v2.2d, #90
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: fmov v6.2d, #3.00000000
+; CHECK-NEXT: fmov v7.2d, #11.00000000
+; CHECK-NEXT: fcmla v4.2d, v0.2d, v2.2d, #0
+; CHECK-NEXT: fcmla v5.2d, v1.2d, v3.2d, #0
+; CHECK-NEXT: fcmla v4.2d, v0.2d, v2.2d, #90
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: fcmla v5.2d, v1.2d, v3.2d, #90
+; CHECK-NEXT: zip2 v1.2d, v7.2d, v6.2d
+; CHECK-NEXT: zip1 v3.2d, v7.2d, v6.2d
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: fcmla v4.2d, v6.2d, v1.2d, #0
-; CHECK-NEXT: fcmla v0.2d, v5.2d, v1.2d, #0
-; CHECK-NEXT: fcmla v4.2d, v6.2d, v1.2d, #90
-; CHECK-NEXT: fcmla v0.2d, v5.2d, v1.2d, #90
-; CHECK-NEXT: mov v1.16b, v4.16b
+; CHECK-NEXT: fcmla v2.2d, v5.2d, v1.2d, #0
+; CHECK-NEXT: fcmla v0.2d, v4.2d, v3.2d, #0
+; CHECK-NEXT: fcmla v2.2d, v5.2d, v1.2d, #90
+; CHECK-NEXT: fcmla v0.2d, v4.2d, v3.2d, #90
+; CHECK-NEXT: mov v1.16b, v2.16b
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 2>
@@ -56,18 +58,21 @@ define <4 x double> @complex_mul_non_const(<4 x double> %a, <4 x double> %b, [2
; CHECK-NEXT: movi v6.2d, #0000000000000000
; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
-; CHECK-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: dup v5.2d, v5.d[0]
+; CHECK-NEXT: dup v4.2d, v4.d[0]
; CHECK-NEXT: fcmla v7.2d, v1.2d, v3.2d, #0
; CHECK-NEXT: fcmla v6.2d, v0.2d, v2.2d, #0
+; CHECK-NEXT: zip2 v16.2d, v4.2d, v5.2d
+; CHECK-NEXT: zip1 v5.2d, v4.2d, v5.2d
+; CHECK-NEXT: movi v4.2d, #0000000000000000
; CHECK-NEXT: fcmla v7.2d, v1.2d, v3.2d, #90
; CHECK-NEXT: fcmla v6.2d, v0.2d, v2.2d, #90
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: fcmla v5.2d, v7.2d, v4.2d, #0
-; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #0
-; CHECK-NEXT: fcmla v5.2d, v7.2d, v4.2d, #90
-; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #90
-; CHECK-NEXT: mov v1.16b, v5.16b
+; CHECK-NEXT: fcmla v4.2d, v7.2d, v16.2d, #0
+; CHECK-NEXT: fcmla v0.2d, v6.2d, v5.2d, #0
+; CHECK-NEXT: fcmla v4.2d, v7.2d, v16.2d, #90
+; CHECK-NEXT: fcmla v0.2d, v6.2d, v5.2d, #90
+; CHECK-NEXT: mov v1.16b, v4.16b
; CHECK-NEXT: ret
entry:
%c.coerce.fca.1.extract = extractvalue [2 x double] %c, 1
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
index 719e5dc67336e..2420164619be0 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
@@ -3,19 +3,12 @@
; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
define {<2 x half>, <2 x half>} @vector_deinterleave_v2f16_v4f16(<4 x half> %vec) {
-; CHECK-SD-LABEL: vector_deinterleave_v2f16_v4f16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uzp1 v2.4h, v0.4h, v0.4h
-; CHECK-SD-NEXT: uzp2 v1.4h, v0.4h, v0.4h
-; CHECK-SD-NEXT: fmov d0, d2
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: vector_deinterleave_v2f16_v4f16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: uzp1 v2.4h, v0.4h, v0.4h
-; CHECK-GI-NEXT: uzp2 v1.4h, v0.4h, v0.4h
-; CHECK-GI-NEXT: fmov d0, d2
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: vector_deinterleave_v2f16_v4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uzp1 v2.4h, v0.4h, v0.4h
+; CHECK-NEXT: uzp2 v1.4h, v0.4h, v0.4h
+; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: ret
%retval = call {<2 x half>, <2 x half>} @llvm.vector.deinterleave2.v4f16(<4 x half> %vec)
ret {<2 x half>, <2 x half>} %retval
}
@@ -46,9 +39,9 @@ define {<8 x half>, <8 x half>} @vector_deinterleave_v8f16_v16f16(<16 x half> %v
define {<2 x float>, <2 x float>} @vector_deinterleave_v2f32_v4f32(<4 x float> %vec) {
; CHECK-SD-LABEL: vector_deinterleave_v2f32_v4f32:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: uzp1 v2.4s, v0.4s, v0.4s
-; CHECK-SD-NEXT: uzp2 v1.4s, v0.4s, v0.4s
-; CHECK-SD-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT: mov d1, v0.d[1]
+; CHECK-SD-NEXT: uzp1 v2.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: uzp2 v1.2s, v0.2s, v1.2s
; CHECK-SD-NEXT: fmov d0, d2
; CHECK-SD-NEXT: ret
;
@@ -75,12 +68,19 @@ ret {<4 x float>, <4 x float>} %retval
}
define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double> %vec) {
-; CHECK-LABEL: vector_deinterleave_v2f64_v4f64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-NEXT: zip2 v1.2d, v0.2d, v1.2d
-; CHECK-NEXT: mov v0.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vector_deinterleave_v2f64_v4f64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vector_deinterleave_v2f64_v4f64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-NEXT: ret
%retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec)
ret {<2 x double>, <2 x double>} %retval
}
@@ -121,12 +121,19 @@ define {<4 x i32>, <4 x i32>} @vector_deinterleave_v4i32_v8i32(<8 x i32> %vec) {
}
define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) {
-; CHECK-LABEL: vector_deinterleave_v2i64_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: zip1 v2.2d, v0.2d, v1.2d
-; CHECK-NEXT: zip2 v1.2d, v0.2d, v1.2d
-; CHECK-NEXT: mov v0.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vector_deinterleave_v2i64_v4i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vector_deinterleave_v2i64_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-NEXT: ret
%retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec)
ret {<2 x i64>, <2 x i64>} %retval
}
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
index 2e4f3757c9089..cc67c054f89c4 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
@@ -12,22 +12,12 @@ define <4 x half> @interleave2_v4f16(<2 x half> %vec0, <2 x half> %vec1) {
}
define <8 x half> @interleave2_v8f16(<4 x half> %vec0, <4 x half> %vec1) {
-; CHECK-SD-LABEL: interleave2_v8f16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT: adrp x8, .LCPI1_0
-; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]
-; CHECK-SD-NEXT: tbl v0.16b, { v0.16b }, v1.16b
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: interleave2_v8f16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT: zip1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: interleave2_v8f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ret
%retval = call <8 x half> @llvm.vector.interleave2.v8f16(<4 x half> %vec0, <4 x half> %vec1)
ret <8 x half> %retval
}
@@ -44,21 +34,12 @@ define <16 x half> @interleave2_v16f16(<8 x half> %vec0, <8 x half> %vec1) {
}
define <4 x float> @interleave2_v4f32(<2 x float> %vec0, <2 x float> %vec1) {
-; CHECK-SD-LABEL: interleave2_v4f32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT: rev64 v1.4s, v0.4s
-; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: interleave2_v4f32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: interleave2_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ret
%retval = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %vec0, <2 x float> %vec1)
ret <4 x float> %retval
}
@@ -149,9 +130,10 @@ define <4 x i16> @interleave2_same_const_splat_v4i16() {
define <4 x i16> @interleave2_diff_const_splat_v4i16() {
; CHECK-SD-LABEL: interleave2_diff_const_splat_v4i16:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mov x8, #1125899907104768 // =0x4000000040000
-; CHECK-SD-NEXT: orr x8, x8, #0x300000003
-; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: movi v0.2s, #4
+; CHECK-SD-NEXT: movi v1.2s, #3
+; CHECK-SD-NEXT: zip1 v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: interleave2_diff_const_splat_v4i16:
@@ -184,12 +166,10 @@ define <4 x i16> @interleave2_same_nonconst_splat_v4i16(i16 %a) {
define <4 x i16> @interleave2_diff_nonconst_splat_v4i16(i16 %a, i16 %b) {
; CHECK-SD-LABEL: interleave2_diff_nonconst_splat_v4i16:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: fmov s0, w0
-; CHECK-SD-NEXT: mov v0.h[1], w0
-; CHECK-SD-NEXT: mov v0.h[2], w1
-; CHECK-SD-NEXT: mov v0.h[3], w1
-; CHECK-SD-NEXT: rev32 v1.4h, v0.4h
-; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: dup v0.2s, w0
+; CHECK-SD-NEXT: dup v1.2s, w1
+; CHECK-SD-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: interleave2_diff_nonconst_splat_v4i16:
diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
index da917119c6b25..d8ff031239b41 100644
--- a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll
@@ -5,25 +5,17 @@
define { <16 x i8>, <16 x i8> } @foo_ld2_v16i8(<16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 v1.16b, v0.16b, v0.16b
; CHECK-NEXT: zip1 v0.16b, v0.16b, v0.16b
; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
; CHECK-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-NEXT: cmlt v1.16b, v1.16b, #0
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-NEXT: and v1.16b, v1.16b, v2.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: umov w9, v1.h[0]
; CHECK-NEXT: umov w8, v0.h[0]
-; CHECK-NEXT: bfi w8, w9, #16, #16
+; CHECK-NEXT: bfi w8, w8, #16, #16
; CHECK-NEXT: tbz w8, #0, .LBB0_2
; CHECK-NEXT: // %bb.1: // %cond.load
; CHECK-NEXT: ldr b1, [x0]
@@ -349,10 +341,11 @@ define { <8 x i16>, <8 x i16> } @foo_ld2_v8i16(<8 x i1> %mask, ptr %p) {
define { <4 x float>, <4 x float> } @foo_ld2_v4f32(<4 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_v4f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v0.8h
; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
-; CHECK-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: shl v0.8b, v0.8b, #7
; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
@@ -419,10 +412,11 @@ define { <4 x float>, <4 x float> } @foo_ld2_v4f32(<4 x i1> %mask, ptr %p) {
define { <2 x double>, <2 x double> } @foo_ld2_v2f64(<2 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_v2f64:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0]
-; CHECK-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: shl v0.4h, v0.4h, #15
; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
@@ -453,8 +447,8 @@ define { <2 x double>, <2 x double> } @foo_ld2_v2f64(<2 x i1> %mask, ptr %p) {
; CHECK-NEXT: add x8, x0, #24
; CHECK-NEXT: ld1 { v2.d }[1], [x8]
; CHECK-NEXT: .LBB3_8: // %else8
-; CHECK-NEXT: zip1 v0.2d, v1.2d, v2.2d
-; CHECK-NEXT: zip2 v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: uzp1 v0.2d, v1.2d, v2.2d
+; CHECK-NEXT: uzp2 v1.2d, v1.2d, v2.2d
; CHECK-NEXT: ret
%interleaved.mask = call <4 x i1> @llvm.vector.interleave2.v4i1(<2 x i1> %mask, <2 x i1> %mask)
%wide.masked.vec = call <4 x double> @llvm.masked.load.v4f64.p0(ptr %p, i32 8, <4 x i1> %interleaved.mask, <4 x double> poison)
diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
index 972be0fca2089..5155be8cfed0d 100644
--- a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll
@@ -5,26 +5,18 @@
define void @foo_st2_v16i8(<16 x i1> %mask, <16 x i8> %val1, <16 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 v3.16b, v0.16b, v0.16b
; CHECK-NEXT: zip1 v0.16b, v0.16b, v0.16b
; CHECK-NEXT: adrp x8, .LCPI0_0
-; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI0_0]
-; CHECK-NEXT: shl v3.16b, v3.16b, #7
+; CHECK-NEXT: ldr q3, [x8, :lo12:.LCPI0_0]
; CHECK-NEXT: shl v0.16b, v0.16b, #7
-; CHECK-NEXT: cmlt v3.16b, v3.16b, #0
; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
-; CHECK-NEXT: and v3.16b, v3.16b, v4.16b
-; CHECK-NEXT: and v0.16b, v0.16b, v4.16b
-; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
+; CHECK-NEXT: and v0.16b, v0.16b, v3.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b
; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b
-; CHECK-NEXT: umov w9, v3.h[0]
; CHECK-NEXT: umov w8, v0.h[0]
; CHECK-NEXT: zip1 v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: bfi w8, w9, #16, #16
+; CHECK-NEXT: bfi w8, w8, #16, #16
; CHECK-NEXT: tbnz w8, #0, .LBB0_33
; CHECK-NEXT: // %bb.1: // %else
; CHECK-NEXT: tbnz w8, #1, .LBB0_34
@@ -345,10 +337,11 @@ define void @foo_st2_v8i16(<8 x i1> %mask, <8 x i16> %val1, <8 x i16> %val2, ptr
define void @foo_st2_v4i32(<4 x i1> %mask, <4 x i32> %val1, <4 x i32> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: zip1 v0.8h, v0.8h, v0.8h
; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI2_0]
-; CHECK-NEXT: zip1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT: xtn v0.8b, v0.8h
; CHECK-NEXT: shl v0.8b, v0.8b, #7
; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
; CHECK-NEXT: and v0.8b, v0.8b, v3.8b
@@ -413,10 +406,11 @@ define void @foo_st2_v4i32(<4 x i1> %mask, <4 x i32> %val1, <4 x i32> %val2, ptr
define void @foo_st2_v2i64(<2 x i1> %mask, <2 x i64> %val1, <2 x i64> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v0.4s
; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI3_0]
-; CHECK-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-NEXT: xtn v0.4h, v0.4s
; CHECK-NEXT: shl v0.4h, v0.4h, #15
; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
; CHECK-NEXT: and v0.8b, v0.8b, v3.8b
>From 61a3b201fd4d52d5a91edeaf08693e83944725d9 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Thu, 9 Jul 2026 10:11:35 +0000
Subject: [PATCH 3/4] [AArch64] Pre-commit tests for interleave/deinterleave
DAG combine
---
.../AArch64/vector-deinterleave-load.ll | 186 ++++++++++++++++++
.../AArch64/vector-interleave-store.ll | 104 ++++++++++
2 files changed, 290 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
create mode 100644 llvm/test/CodeGen/AArch64/vector-interleave-store.ll
diff --git a/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll b/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
new file mode 100644
index 0000000000000..c79dc0624b5c3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,IA-ENABLE
+; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,IA-DISABLE
+
+define void @aarch64_vector_deinterleave_idx_ld2(ptr %ptr, i64 %idx) {
+; IA-ENABLE-LABEL: aarch64_vector_deinterleave_idx_ld2:
+; IA-ENABLE: // %bb.0: // %entry
+; IA-ENABLE-NEXT: lsl x8, x1, #2
+; IA-ENABLE-NEXT: and x9, x8, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: add x8, x8, #16
+; IA-ENABLE-NEXT: add x10, x0, x9
+; IA-ENABLE-NEXT: and x8, x8, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: ld2 { v0.4s, v1.4s }, [x10]
+; IA-ENABLE-NEXT: str q0, [x0, x9]
+; IA-ENABLE-NEXT: str q1, [x0, x8]
+; IA-ENABLE-NEXT: ret
+;
+; IA-DISABLE-LABEL: aarch64_vector_deinterleave_idx_ld2:
+; IA-DISABLE: // %bb.0: // %entry
+; IA-DISABLE-NEXT: lsl x8, x1, #2
+; IA-DISABLE-NEXT: and x9, x8, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: add x8, x8, #16
+; IA-DISABLE-NEXT: add x9, x0, x9
+; IA-DISABLE-NEXT: and x8, x8, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: ldp q1, q0, [x9]
+; IA-DISABLE-NEXT: uzp1 v2.4s, v1.4s, v0.4s
+; IA-DISABLE-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; IA-DISABLE-NEXT: str q2, [x9]
+; IA-DISABLE-NEXT: str q0, [x0, x8]
+; IA-DISABLE-NEXT: ret
+entry:
+ %idx1 = lshr i64 %idx, 2
+ %a1 = add i64 %idx, 4
+ %idx2 = lshr i64 %a1, 2
+
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx2
+
+ %load = load <8 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float> } @llvm.vector.deinterleave2.v8f32(<8 x float> %load)
+ %m0_3 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float> } %deinterleave, 1
+
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ ret void
+}
+
+define void @aarch64_vector_deinterleave_idx_ld3(ptr %ptr, i64 %idx) {
+; IA-ENABLE-LABEL: aarch64_vector_deinterleave_idx_ld3:
+; IA-ENABLE: // %bb.0: // %entry
+; IA-ENABLE-NEXT: lsl x8, x1, #2
+; IA-ENABLE-NEXT: and x9, x8, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: add x10, x0, x9
+; IA-ENABLE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x10]
+; IA-ENABLE-NEXT: add x10, x8, #16
+; IA-ENABLE-NEXT: add x8, x8, #32
+; IA-ENABLE-NEXT: and x10, x10, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: and x8, x8, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: str q0, [x0, x9]
+; IA-ENABLE-NEXT: str q1, [x0, x10]
+; IA-ENABLE-NEXT: str q2, [x0, x8]
+; IA-ENABLE-NEXT: ret
+;
+; IA-DISABLE-LABEL: aarch64_vector_deinterleave_idx_ld3:
+; IA-DISABLE: // %bb.0: // %entry
+; IA-DISABLE-NEXT: sub sp, sp, #48
+; IA-DISABLE-NEXT: .cfi_def_cfa_offset 48
+; IA-DISABLE-NEXT: lsl x9, x1, #2
+; IA-DISABLE-NEXT: mov x8, sp
+; IA-DISABLE-NEXT: and x10, x9, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: add x10, x0, x10
+; IA-DISABLE-NEXT: ldp q1, q0, [x10, #16]
+; IA-DISABLE-NEXT: ldr q2, [x10]
+; IA-DISABLE-NEXT: str q2, [sp]
+; IA-DISABLE-NEXT: stp q1, q0, [sp, #16]
+; IA-DISABLE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x8]
+; IA-DISABLE-NEXT: add x8, x9, #16
+; IA-DISABLE-NEXT: add x9, x9, #32
+; IA-DISABLE-NEXT: and x8, x8, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: and x9, x9, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: str q0, [x10]
+; IA-DISABLE-NEXT: str q1, [x0, x8]
+; IA-DISABLE-NEXT: str q2, [x0, x9]
+; IA-DISABLE-NEXT: add sp, sp, #48
+; IA-DISABLE-NEXT: ret
+entry:
+ %idx1 = lshr i64 %idx, 2
+ %a1 = add i64 %idx, 4
+ %idx2 = lshr i64 %a1, 2
+ %a2 = add i64 %idx, 8
+ %idx3 = lshr i64 %a2, 2
+
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx2
+ %gep3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx3
+
+ %load = load <12 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave3.v12f32(<12 x float> %load)
+ %m0_3 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleave, 1
+ %m8_11 = extractvalue { <4 x float>, <4 x float>, <4 x float> } %deinterleave, 2
+
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ store <4 x float> %m8_11, ptr %gep3, align 16
+ ret void
+}
+
+define void @aarch64_vector_deinterleave_idx_ld4(ptr %ptr, i64 %idx) {
+; IA-ENABLE-LABEL: aarch64_vector_deinterleave_idx_ld4:
+; IA-ENABLE: // %bb.0: // %entry
+; IA-ENABLE-NEXT: lsl x8, x1, #2
+; IA-ENABLE-NEXT: add x9, x8, #64
+; IA-ENABLE-NEXT: add x11, x8, #96
+; IA-ENABLE-NEXT: and x9, x9, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: add x10, x0, x9
+; IA-ENABLE-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x10]
+; IA-ENABLE-NEXT: add x10, x8, #80
+; IA-ENABLE-NEXT: add x8, x8, #112
+; IA-ENABLE-NEXT: and x10, x10, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: and x8, x8, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: str q0, [x0, x9]
+; IA-ENABLE-NEXT: and x9, x11, #0xfffffffffffffff0
+; IA-ENABLE-NEXT: str q1, [x0, x10]
+; IA-ENABLE-NEXT: str q2, [x0, x9]
+; IA-ENABLE-NEXT: str q3, [x0, x8]
+; IA-ENABLE-NEXT: ret
+;
+; IA-DISABLE-LABEL: aarch64_vector_deinterleave_idx_ld4:
+; IA-DISABLE: // %bb.0: // %entry
+; IA-DISABLE-NEXT: lsl x8, x1, #2
+; IA-DISABLE-NEXT: add x9, x8, #64
+; IA-DISABLE-NEXT: add x10, x8, #80
+; IA-DISABLE-NEXT: add x11, x8, #96
+; IA-DISABLE-NEXT: and x9, x9, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: add x8, x8, #112
+; IA-DISABLE-NEXT: and x10, x10, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: add x9, x0, x9
+; IA-DISABLE-NEXT: and x8, x8, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: ldp q1, q0, [x9]
+; IA-DISABLE-NEXT: ldp q3, q2, [x9, #32]
+; IA-DISABLE-NEXT: uzp1 v5.4s, v1.4s, v0.4s
+; IA-DISABLE-NEXT: uzp2 v0.4s, v1.4s, v0.4s
+; IA-DISABLE-NEXT: uzp1 v4.4s, v3.4s, v2.4s
+; IA-DISABLE-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; IA-DISABLE-NEXT: uzp1 v1.4s, v5.4s, v4.4s
+; IA-DISABLE-NEXT: uzp1 v3.4s, v0.4s, v2.4s
+; IA-DISABLE-NEXT: uzp2 v4.4s, v5.4s, v4.4s
+; IA-DISABLE-NEXT: uzp2 v0.4s, v0.4s, v2.4s
+; IA-DISABLE-NEXT: str q1, [x9]
+; IA-DISABLE-NEXT: and x9, x11, #0xfffffffffffffff0
+; IA-DISABLE-NEXT: str q3, [x0, x10]
+; IA-DISABLE-NEXT: str q4, [x0, x9]
+; IA-DISABLE-NEXT: str q0, [x0, x8]
+; IA-DISABLE-NEXT: ret
+entry:
+ %a1 = add i64 %idx, 16
+ %idx1 = lshr i64 %a1, 2
+ %a2 = add i64 %idx, 20
+ %idx2 = lshr i64 %a2, 2
+ %a3 = add i64 %idx, 24
+ %idx3 = lshr i64 %a3, 2
+ %a4 = add i64 %idx, 28
+ %idx4 = lshr i64 %a4, 2
+
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+ %gep2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx2
+ %gep3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx3
+ %gep4 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx4
+
+ %load = load <16 x float>, ptr %gep1, align 16
+ %deinterleave = call { <4 x float>, <4 x float>, <4 x float>, <4 x float> } @llvm.vector.deinterleave4.v16f32(<16 x float> %load)
+ %m0_3 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 0
+ %m4_7 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 1
+ %m8_11 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 2
+ %m12_15 = extractvalue { <4 x float>, <4 x float>, <4 x float>, <4 x float> } %deinterleave, 3
+
+ store <4 x float> %m0_3, ptr %gep1, align 16
+ store <4 x float> %m4_7, ptr %gep2, align 16
+ store <4 x float> %m8_11, ptr %gep3, align 16
+ store <4 x float> %m12_15, ptr %gep4, align 16
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
new file mode 100644
index 0000000000000..c5edb6cf82984
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,IA-ENABLE
+; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,IA-DISABLE
+define void @aarch64_vector_interleave_idx_st2(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1) {
+; IA-ENABLE-LABEL: aarch64_vector_interleave_idx_st2:
+; IA-ENABLE: // %bb.0: // %entry
+; IA-ENABLE-NEXT: lsr x8, x1, #2
+; IA-ENABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; IA-ENABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; IA-ENABLE-NEXT: add x8, x0, x8, lsl #4
+; IA-ENABLE-NEXT: st2 { v0.4s, v1.4s }, [x8]
+; IA-ENABLE-NEXT: ret
+;
+; IA-DISABLE-LABEL: aarch64_vector_interleave_idx_st2:
+; IA-DISABLE: // %bb.0: // %entry
+; IA-DISABLE-NEXT: lsr x8, x1, #2
+; IA-DISABLE-NEXT: zip2 v2.4s, v0.4s, v1.4s
+; IA-DISABLE-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; IA-DISABLE-NEXT: add x8, x0, x8, lsl #4
+; IA-DISABLE-NEXT: stp q0, q2, [x8]
+; IA-DISABLE-NEXT: ret
+entry:
+ %idx1 = lshr i64 %idx, 2
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+
+ %interleave = call <8 x float> @llvm.vector.interleave2.v8f32(<4 x float> %v0, <4 x float> %v1)
+ store <8 x float> %interleave, ptr %gep1, align 16
+ ret void
+}
+
+define void @aarch64_vector_interleave_idx_st3(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
+; IA-ENABLE-LABEL: aarch64_vector_interleave_idx_st3:
+; IA-ENABLE: // %bb.0: // %entry
+; IA-ENABLE-NEXT: lsr x8, x1, #2
+; IA-ENABLE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; IA-ENABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; IA-ENABLE-NEXT: add x8, x0, x8, lsl #4
+; IA-ENABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; IA-ENABLE-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
+; IA-ENABLE-NEXT: ret
+;
+; IA-DISABLE-LABEL: aarch64_vector_interleave_idx_st3:
+; IA-DISABLE: // %bb.0: // %entry
+; IA-DISABLE-NEXT: sub sp, sp, #48
+; IA-DISABLE-NEXT: .cfi_def_cfa_offset 48
+; IA-DISABLE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; IA-DISABLE-NEXT: mov x8, sp
+; IA-DISABLE-NEXT: lsr x9, x1, #2
+; IA-DISABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; IA-DISABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; IA-DISABLE-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
+; IA-DISABLE-NEXT: add x8, x0, x9, lsl #4
+; IA-DISABLE-NEXT: ldp q1, q0, [sp, #16]
+; IA-DISABLE-NEXT: ldr q2, [sp]
+; IA-DISABLE-NEXT: stp q1, q0, [x8, #16]
+; IA-DISABLE-NEXT: str q2, [x8]
+; IA-DISABLE-NEXT: add sp, sp, #48
+; IA-DISABLE-NEXT: ret
+entry:
+ %idx1 = lshr i64 %idx, 2
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+
+ %interleave = call <12 x float> @llvm.vector.interleave3.v12f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2)
+ store <12 x float> %interleave, ptr %gep1, align 16
+ ret void
+}
+
+define void @aarch64_vector_interleave_idx_st4(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1, <4 x float> %v2, <4 x float> %v3) {
+; IA-ENABLE-LABEL: aarch64_vector_interleave_idx_st4:
+; IA-ENABLE: // %bb.0: // %entry
+; IA-ENABLE-NEXT: lsr x8, x1, #2
+; IA-ENABLE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; IA-ENABLE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; IA-ENABLE-NEXT: add x8, x0, x8, lsl #4
+; IA-ENABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; IA-ENABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; IA-ENABLE-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
+; IA-ENABLE-NEXT: ret
+;
+; IA-DISABLE-LABEL: aarch64_vector_interleave_idx_st4:
+; IA-DISABLE: // %bb.0: // %entry
+; IA-DISABLE-NEXT: zip2 v4.4s, v1.4s, v3.4s
+; IA-DISABLE-NEXT: zip2 v5.4s, v0.4s, v2.4s
+; IA-DISABLE-NEXT: lsr x8, x1, #2
+; IA-DISABLE-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; IA-DISABLE-NEXT: zip1 v0.4s, v0.4s, v2.4s
+; IA-DISABLE-NEXT: add x8, x0, x8, lsl #4
+; IA-DISABLE-NEXT: zip2 v2.4s, v5.4s, v4.4s
+; IA-DISABLE-NEXT: zip1 v3.4s, v5.4s, v4.4s
+; IA-DISABLE-NEXT: zip2 v4.4s, v0.4s, v1.4s
+; IA-DISABLE-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; IA-DISABLE-NEXT: stp q3, q2, [x8, #32]
+; IA-DISABLE-NEXT: stp q0, q4, [x8]
+; IA-DISABLE-NEXT: ret
+entry:
+ %idx1 = lshr i64 %idx, 2
+ %gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
+
+ %interleave = call <16 x float> @llvm.vector.interleave4.v16f32(<4 x float> %v0, <4 x float> %v1, <4 x float> %v2, <4 x float> %v3)
+ store <16 x float> %interleave, ptr %gep1, align 16
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
>From 5849ab65671843f6a566dab01907ce0630cc3fb4 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Sat, 11 Jul 2026 15:02:46 +0100
Subject: [PATCH 4/4] [AArch64] Add DAG combines for fixed width (de)interleave
---
.../Target/AArch64/AArch64ISelLowering.cpp | 220 ++++++++++++------
.../AArch64/fixed-vector-deinterleave.ll | 46 ++++
.../AArch64/fixed-vector-interleave.ll | 126 +++++-----
.../scalable_masked_interleaved_stores.ll | 9 +-
.../AArch64/vector-deinterleave-load.ll | 154 ++++--------
.../AArch64/vector-interleave-store.ll | 100 +++-----
6 files changed, 341 insertions(+), 314 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f33d953eec747..cee345bbd2a7b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27258,6 +27258,10 @@ static unsigned getFPSubregForVT(EVT VT) {
}
}
+static SDValue
+performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG);
+
static SDValue performSTORECombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
@@ -27289,6 +27293,9 @@ static SDValue performSTORECombine(SDNode *N,
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
+ if (SDValue Res = performInterleavedStoreCombine(N, DCI, DAG))
+ return Res;
+
auto hasValidElementTypeForFPTruncStore = [](EVT VT) {
EVT EltVT = VT.getVectorElementType();
return EltVT == MVT::f32 || EltVT == MVT::f64;
@@ -27471,19 +27478,36 @@ static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
WideMask->getOperand(0));
}
-static SDValue performInterleavedMaskedStoreCombine(
- SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
+static SDValue
+performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG) {
if (!DCI.isBeforeLegalize())
return SDValue();
- MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
- SDValue WideValue = MST->getValue();
+ SDValue WideValue, Chain, BasePtr, Mask;
+ bool IsMasked = false;
+ if (auto *MST = dyn_cast<MaskedStoreSDNode>(N)) {
+ if (!ISD::isNormalMaskedStore(MST) || !MST->isSimple() ||
+ !MST->getOffset().isUndef())
+ return SDValue();
+ WideValue = MST->getValue();
+ Chain = MST->getChain();
+ BasePtr = MST->getBasePtr();
+ Mask = MST->getMask();
+ IsMasked = true;
+ } else {
+ auto *ST = cast<StoreSDNode>(N);
+ if (!ISD::isNormalStore(ST) || !ST->isSimple() ||
+ !ST->getOffset().isUndef())
+ return SDValue();
+ WideValue = ST->getValue();
+ Chain = ST->getChain();
+ BasePtr = ST->getBasePtr();
+ }
// Bail out if the stored value has an unexpected number of uses, since we'll
- // have to perform manual interleaving and may as well just use normal masked
- // stores. Also, discard masked stores that are truncating or indexed.
- if (!WideValue.hasOneUse() || !ISD::isNormalMaskedStore(MST) ||
- !MST->isSimple() || !MST->getOffset().isUndef())
+ // have to perform manual interleaving and may as well just use normal stores.
+ if (!WideValue.hasOneUse())
return SDValue();
SmallVector<SDValue, 4> ValueInterleaveOps;
@@ -27492,30 +27516,59 @@ static SDValue performInterleavedMaskedStoreCombine(
return SDValue();
unsigned NumParts = ValueInterleaveOps.size();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
- // At the moment we're unlikely to see a fixed-width vector interleave as
- // we usually generate shuffles instead.
EVT SubVecTy = ValueInterleaveOps[0].getValueType();
- if (!SubVecTy.isScalableVT() ||
- SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
- !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ if (!TLI.isTypeLegal(SubVecTy))
return SDValue();
+ bool IsScalable = SubVecTy.isScalableVector();
+ unsigned SubBits = SubVecTy.getSizeInBits().getKnownMinValue();
+ if (IsScalable) {
+ if (SubBits != 128)
+ return SDValue();
+ } else if (SubBits != 64 && SubBits != 128) {
+ return SDValue();
+ }
SDLoc DL(N);
- SDValue NarrowMask =
- getNarrowMaskForInterleavedOps(DAG, DL, MST->getMask(), NumParts);
- if (!NarrowMask)
- return SDValue();
+ if (IsScalable) {
+ if (NumParts == 3)
+ return SDValue();
+ SDValue Pred;
+ if (IsMasked) {
+ Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
+ if (!Pred)
+ return SDValue();
+ } else {
+ EVT PredVT = SubVecTy.changeVectorElementType(*DAG.getContext(), MVT::i1);
+ Pred = DAG.getConstant(1, DL, PredVT);
+ }
+
+ const Intrinsic::ID IID =
+ NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
+ SmallVector<SDValue, 8> Ops;
+ Ops.append({Chain, DAG.getConstant(IID, DL, MVT::i32)});
+ Ops.append(ValueInterleaveOps);
+ Ops.append({Pred, BasePtr});
+ return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops);
+ } else {
- const Intrinsic::ID IID =
- NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
- SmallVector<SDValue, 8> NewStOps;
- NewStOps.append({MST->getChain(), DAG.getConstant(IID, DL, MVT::i32)});
- NewStOps.append(ValueInterleaveOps);
- NewStOps.append({NarrowMask, MST->getBasePtr()});
- return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, NewStOps);
+ static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2,
+ Intrinsic::aarch64_neon_st3,
+ Intrinsic::aarch64_neon_st4};
+ SmallVector<SDValue, 8> Ops;
+ Ops.push_back(Chain);
+ Ops.push_back(
+ DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64));
+ Ops.append(ValueInterleaveOps);
+ Ops.push_back(BasePtr);
+ auto *MemN = cast<MemSDNode>(N);
+ return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
+ DAG.getVTList(MVT::Other), Ops,
+ MemN->getMemoryVT(), MemN->getMemOperand());
+ }
}
static SDValue performMSTORECombine(SDNode *N,
@@ -27527,7 +27580,7 @@ static SDValue performMSTORECombine(SDNode *N,
SDValue Mask = MST->getMask();
SDLoc DL(N);
- if (SDValue Res = performInterleavedMaskedStoreCombine(N, DCI, DAG))
+ if (SDValue Res = performInterleavedStoreCombine(N, DCI, DAG))
return Res;
// If this is a UZP1 followed by a masked store, fold this into a masked
@@ -30398,21 +30451,26 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
unsigned NumParts = N->getNumOperands();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
EVT SubVecTy = N->getValueType(0);
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- // At the moment we're unlikely to see a fixed-width vector deinterleave as
- // we usually generate shuffles instead.
- unsigned MinNumElements = SubVecTy.getVectorMinNumElements();
- if (!SubVecTy.isScalableVector() ||
- SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
- !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ bool IsScalable = SubVecTy.isScalableVector();
+ unsigned SubVecBits = SubVecTy.getSizeInBits().getKnownMinValue();
+ if (!TLI.isTypeLegal(SubVecTy))
return SDValue();
+ if (IsScalable) {
+ if (SubVecBits != 128)
+ return SDValue();
+ } else if (SubVecBits != 64 && SubVecBits != 128) {
+ return SDValue();
+ }
// Make sure each input operand is the correct extract_subvector of the same
// wider vector.
+ unsigned MinNumElements = SubVecTy.getVectorMinNumElements();
SDValue Op0 = N->getOperand(0);
for (unsigned I = 0; I < NumParts; I++) {
SDValue OpI = N->getOperand(I);
@@ -30423,40 +30481,72 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
}
- // Normal loads are currently already handled by the InterleavedAccessPass so
- // we don't expect to see them here. Bail out if the masked load has an
- // unexpected number of uses, since we want to avoid a situation where we have
- // both deinterleaving loads and normal loads in the same block. Also, discard
- // masked loads that are extending, indexed, have an unexpected offset or have
- // an unsupported passthru value until we find a valid use case.
- auto MaskedLoad = dyn_cast<MaskedLoadSDNode>(Op0->getOperand(0));
- if (!MaskedLoad || !MaskedLoad->hasNUsesOfValue(NumParts, 0) ||
- !MaskedLoad->isSimple() || !ISD::isNormalMaskedLoad(MaskedLoad) ||
- !MaskedLoad->getOffset().isUndef() ||
- (!MaskedLoad->getPassThru()->isUndef() &&
- !isZerosVector(MaskedLoad->getPassThru().getNode())))
- return SDValue();
-
- // Now prove that the mask is an interleave of identical masks.
+ SDValue WideVec = Op0->getOperand(0);
SDLoc DL(N);
- SDValue NarrowMask =
- getNarrowMaskForInterleavedOps(DAG, DL, MaskedLoad->getMask(), NumParts);
- if (!NarrowMask)
- return SDValue();
- const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
- : Intrinsic::aarch64_sve_ld4_sret;
- SDValue NewLdOps[] = {MaskedLoad->getChain(),
- DAG.getConstant(IID, DL, MVT::i32), NarrowMask,
- MaskedLoad->getBasePtr()};
+ SmallVector<EVT, 5> ResVTs(NumParts, SubVecTy);
+ ResVTs.push_back(MVT::Other);
+ SDVTList ResVTList = DAG.getVTList(ResVTs);
+
SDValue Res;
- if (NumParts == 2)
- Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
- else
- Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, SubVecTy, SubVecTy, MVT::Other},
- NewLdOps);
+ if (IsScalable) {
+ if (NumParts == 3)
+ return SDValue();
+ SDValue Chain, BasePtr, Pred;
+ if (auto *MaskedLoad = dyn_cast<MaskedLoadSDNode>(WideVec)) {
+ // Bail out if the masked load has an unexpected number of uses, since we
+ // want to avoid a situation where we have both deinterleaving loads and
+ // normal loads in the same block. Also, discard masked loads that are
+ // extending, indexed, have an unexpected offset or have an unsupported
+ // passthru value until we find a valid use case.
+ if (!MaskedLoad->hasNUsesOfValue(NumParts, 0) ||
+ !MaskedLoad->isSimple() || !ISD::isNormalMaskedLoad(MaskedLoad) ||
+ !MaskedLoad->getOffset().isUndef() ||
+ (!MaskedLoad->getPassThru()->isUndef() &&
+ !isZerosVector(MaskedLoad->getPassThru().getNode())))
+ return SDValue();
+
+ // Now prove that the mask is an interleave of identical masks.
+ Pred = getNarrowMaskForInterleavedOps(DAG, DL, MaskedLoad->getMask(),
+ NumParts);
+ if (!Pred)
+ return SDValue();
+ Chain = MaskedLoad->getChain();
+ BasePtr = MaskedLoad->getBasePtr();
+ } else {
+ auto *Load = dyn_cast<LoadSDNode>(WideVec);
+ if (!Load || !Load->hasNUsesOfValue(NumParts, 0) || !Load->isSimple() ||
+ !ISD::isNormalLoad(Load) || !Load->getOffset().isUndef())
+ return SDValue();
+
+ EVT PredVT = SubVecTy.changeVectorElementType(*DAG.getContext(), MVT::i1);
+ Pred = DAG.getConstant(1, DL, PredVT);
+ Chain = Load->getChain();
+ BasePtr = Load->getBasePtr();
+ }
+
+ const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
+ : Intrinsic::aarch64_sve_ld4_sret;
+ SDValue NewLdOps[] = {Chain, DAG.getConstant(IID, DL, MVT::i32), Pred,
+ BasePtr};
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTList, NewLdOps);
+ } else {
+ auto *Load = dyn_cast<LoadSDNode>(WideVec);
+ if (!Load || !Load->hasNUsesOfValue(NumParts, 0) || !Load->isSimple() ||
+ !ISD::isNormalLoad(Load) || !Load->getOffset().isUndef())
+ return SDValue();
+
+ static constexpr Intrinsic::ID NEONLoads[] = {Intrinsic::aarch64_neon_ld2,
+ Intrinsic::aarch64_neon_ld3,
+ Intrinsic::aarch64_neon_ld4};
+ SDValue NewLdOps[] = {
+ Load->getChain(),
+ DAG.getTargetConstant(NEONLoads[NumParts - 2], DL, MVT::i64),
+ Load->getBasePtr()};
+ Res =
+ DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTList, NewLdOps,
+ Load->getMemoryVT(), Load->getMemOperand());
+ }
// We can now generate a structured load!
SmallVector<SDValue, 4> ResOps(NumParts);
@@ -30464,7 +30554,7 @@ static SDValue performVectorDeinterleaveCombine(
ResOps[Idx] = SDValue(Res.getNode(), Idx);
// Replace uses of the original chain result with the new chain result.
- DAG.ReplaceAllUsesOfValueWith(SDValue(MaskedLoad, 1),
+ DAG.ReplaceAllUsesOfValueWith(WideVec.getValue(1),
SDValue(Res.getNode(), NumParts));
return DCI.CombineTo(N, ResOps, false);
}
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
index 2420164619be0..f1d86a2380f6d 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IA-DISABLE-SD
+; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false -global-isel -global-isel-abort=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IA-DISABLE-GI
define {<2 x half>, <2 x half>} @vector_deinterleave_v2f16_v4f16(<4 x half> %vec) {
; CHECK-LABEL: vector_deinterleave_v2f16_v4f16:
@@ -52,6 +54,22 @@ define {<2 x float>, <2 x float>} @vector_deinterleave_v2f32_v4f32(<4 x float> %
; CHECK-GI-NEXT: // kill: def $d1 killed $d1 killed $q1
; CHECK-GI-NEXT: fmov d0, d2
; CHECK-GI-NEXT: ret
+;
+; CHECK-IA-DISABLE-SD-LABEL: vector_deinterleave_v2f32_v4f32:
+; CHECK-IA-DISABLE-SD: // %bb.0:
+; CHECK-IA-DISABLE-SD-NEXT: mov d1, v0.d[1]
+; CHECK-IA-DISABLE-SD-NEXT: uzp1 v2.2s, v0.2s, v1.2s
+; CHECK-IA-DISABLE-SD-NEXT: uzp2 v1.2s, v0.2s, v1.2s
+; CHECK-IA-DISABLE-SD-NEXT: fmov d0, d2
+; CHECK-IA-DISABLE-SD-NEXT: ret
+;
+; CHECK-IA-DISABLE-GI-LABEL: vector_deinterleave_v2f32_v4f32:
+; CHECK-IA-DISABLE-GI: // %bb.0:
+; CHECK-IA-DISABLE-GI-NEXT: uzp1 v2.4s, v0.4s, v0.4s
+; CHECK-IA-DISABLE-GI-NEXT: uzp2 v1.4s, v0.4s, v0.4s
+; CHECK-IA-DISABLE-GI-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-IA-DISABLE-GI-NEXT: fmov d0, d2
+; CHECK-IA-DISABLE-GI-NEXT: ret
%retval = call {<2 x float>, <2 x float>} @llvm.vector.deinterleave2.v4f32(<4 x float> %vec)
ret {<2 x float>, <2 x float>} %retval
}
@@ -81,6 +99,20 @@ define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double
; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: ret
+;
+; CHECK-IA-DISABLE-SD-LABEL: vector_deinterleave_v2f64_v4f64:
+; CHECK-IA-DISABLE-SD: // %bb.0:
+; CHECK-IA-DISABLE-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-IA-DISABLE-SD-NEXT: ret
+;
+; CHECK-IA-DISABLE-GI-LABEL: vector_deinterleave_v2f64_v4f64:
+; CHECK-IA-DISABLE-GI: // %bb.0:
+; CHECK-IA-DISABLE-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-GI-NEXT: mov v0.16b, v2.16b
+; CHECK-IA-DISABLE-GI-NEXT: ret
%retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec)
ret {<2 x double>, <2 x double>} %retval
}
@@ -134,6 +166,20 @@ define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) {
; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: ret
+;
+; CHECK-IA-DISABLE-SD-LABEL: vector_deinterleave_v2i64_v4i64:
+; CHECK-IA-DISABLE-SD: // %bb.0:
+; CHECK-IA-DISABLE-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-IA-DISABLE-SD-NEXT: ret
+;
+; CHECK-IA-DISABLE-GI-LABEL: vector_deinterleave_v2i64_v4i64:
+; CHECK-IA-DISABLE-GI: // %bb.0:
+; CHECK-IA-DISABLE-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d
+; CHECK-IA-DISABLE-GI-NEXT: mov v0.16b, v2.16b
+; CHECK-IA-DISABLE-GI-NEXT: ret
%retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec)
ret {<2 x i64>, <2 x i64>} %retval
}
diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
index cc67c054f89c4..8506fa09181a7 100644
--- a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-IA-DISABLE-SD
+; RUN: llc -mtriple=aarch64-none-linux-gnu -lower-interleaved-accesses=false -global-isel -global-isel-abort=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-IA-DISABLE-GI
define <4 x half> @interleave2_v4f16(<2 x half> %vec0, <2 x half> %vec1) {
; CHECK-LABEL: interleave2_v4f16:
@@ -123,6 +125,17 @@ define <4 x i16> @interleave2_same_const_splat_v4i16() {
; CHECK-GI-NEXT: movi v0.4h, #3
; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT: ret
+;
+; CHECK-IA-DISABLE-SD-LABEL: interleave2_same_const_splat_v4i16:
+; CHECK-IA-DISABLE-SD: // %bb.0:
+; CHECK-IA-DISABLE-SD-NEXT: movi v0.4h, #3
+; CHECK-IA-DISABLE-SD-NEXT: ret
+;
+; CHECK-IA-DISABLE-GI-LABEL: interleave2_same_const_splat_v4i16:
+; CHECK-IA-DISABLE-GI: // %bb.0:
+; CHECK-IA-DISABLE-GI-NEXT: movi v0.4h, #3
+; CHECK-IA-DISABLE-GI-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-IA-DISABLE-GI-NEXT: ret
%retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> splat(i16 3), <2 x i16> splat(i16 3))
ret <4 x i16> %retval
}
@@ -142,6 +155,21 @@ define <4 x i16> @interleave2_diff_const_splat_v4i16() {
; CHECK-GI-NEXT: movi v1.4h, #4
; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT: ret
+;
+; CHECK-IA-DISABLE-SD-LABEL: interleave2_diff_const_splat_v4i16:
+; CHECK-IA-DISABLE-SD: // %bb.0:
+; CHECK-IA-DISABLE-SD-NEXT: movi v0.2s, #4
+; CHECK-IA-DISABLE-SD-NEXT: movi v1.2s, #3
+; CHECK-IA-DISABLE-SD-NEXT: zip1 v0.4s, v1.4s, v0.4s
+; CHECK-IA-DISABLE-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-IA-DISABLE-SD-NEXT: ret
+;
+; CHECK-IA-DISABLE-GI-LABEL: interleave2_diff_const_splat_v4i16:
+; CHECK-IA-DISABLE-GI: // %bb.0:
+; CHECK-IA-DISABLE-GI-NEXT: movi v0.4h, #3
+; CHECK-IA-DISABLE-GI-NEXT: movi v1.4h, #4
+; CHECK-IA-DISABLE-GI-NEXT: zip1 v0.4h, v0.4h, v1.4h
+; CHECK-IA-DISABLE-GI-NEXT: ret
%retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> splat(i16 3), <2 x i16> splat(i16 4))
ret <4 x i16> %retval
}
@@ -157,6 +185,17 @@ define <4 x i16> @interleave2_same_nonconst_splat_v4i16(i16 %a) {
; CHECK-GI-NEXT: dup v0.4h, w0
; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v0.4h
; CHECK-GI-NEXT: ret
+;
+; CHECK-IA-DISABLE-SD-LABEL: interleave2_same_nonconst_splat_v4i16:
+; CHECK-IA-DISABLE-SD: // %bb.0:
+; CHECK-IA-DISABLE-SD-NEXT: dup v0.4h, w0
+; CHECK-IA-DISABLE-SD-NEXT: ret
+;
+; CHECK-IA-DISABLE-GI-LABEL: interleave2_same_nonconst_splat_v4i16:
+; CHECK-IA-DISABLE-GI: // %bb.0:
+; CHECK-IA-DISABLE-GI-NEXT: dup v0.4h, w0
+; CHECK-IA-DISABLE-GI-NEXT: zip1 v0.4h, v0.4h, v0.4h
+; CHECK-IA-DISABLE-GI-NEXT: ret
%ins = insertelement <2 x i16> poison, i16 %a, i32 0
%splat = shufflevector <2 x i16> %ins, <2 x i16> poison, <2 x i32> <i32 0, i32 0>
%retval = call <4 x i16> @llvm.vector.interleave2.v4i16(<2 x i16> %splat, <2 x i16> %splat)
@@ -178,6 +217,21 @@ define <4 x i16> @interleave2_diff_nonconst_splat_v4i16(i16 %a, i16 %b) {
; CHECK-GI-NEXT: dup v1.4h, w1
; CHECK-GI-NEXT: zip1 v0.4h, v0.4h, v1.4h
; CHECK-GI-NEXT: ret
+;
+; CHECK-IA-DISABLE-SD-LABEL: interleave2_diff_nonconst_splat_v4i16:
+; CHECK-IA-DISABLE-SD: // %bb.0:
+; CHECK-IA-DISABLE-SD-NEXT: dup v0.2s, w0
+; CHECK-IA-DISABLE-SD-NEXT: dup v1.2s, w1
+; CHECK-IA-DISABLE-SD-NEXT: zip1 v0.4s, v0.4s, v1.4s
+; CHECK-IA-DISABLE-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-IA-DISABLE-SD-NEXT: ret
+;
+; CHECK-IA-DISABLE-GI-LABEL: interleave2_diff_nonconst_splat_v4i16:
+; CHECK-IA-DISABLE-GI: // %bb.0:
+; CHECK-IA-DISABLE-GI-NEXT: dup v0.4h, w0
+; CHECK-IA-DISABLE-GI-NEXT: dup v1.4h, w1
+; CHECK-IA-DISABLE-GI-NEXT: zip1 v0.4h, v0.4h, v1.4h
+; CHECK-IA-DISABLE-GI-NEXT: ret
%ins1 = insertelement <2 x i16> poison, i16 %a, i32 0
%splat1 = shufflevector <2 x i16> %ins1, <2 x i16> poison, <2 x i32> <i32 0, i32 0>
%ins2 = insertelement <2 x i16> poison, i16 %b, i32 0
@@ -455,18 +509,10 @@ define <6 x double> @interleave3_v6f64(<2 x double> %vec0, <2 x double> %vec1, <
define <12 x float> @interleave3_v12f32(<4 x float> %vec0, <4 x float> %vec1, <4 x float> %vec2) {
; CHECK-LABEL: interleave3_v12f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #48
-; CHECK-NEXT: .cfi_def_cfa_offset 48
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x9]
-; CHECK-NEXT: ldp q1, q0, [sp, #16]
-; CHECK-NEXT: ldr q2, [sp]
-; CHECK-NEXT: stp q1, q0, [x8, #16]
-; CHECK-NEXT: str q2, [x8]
-; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
; CHECK-NEXT: ret
%retval = call <12 x float> @llvm.vector.interleave3.v12f32(<4 x float> %vec0, <4 x float> %vec1, <4 x float> %vec2)
ret <12 x float> %retval
@@ -475,18 +521,10 @@ define <12 x float> @interleave3_v12f32(<4 x float> %vec0, <4 x float> %vec1, <4
define <24 x i16> @interleave3_v24i16(<8 x i16> %vec0, <8 x i16> %vec1, <8 x i16> %vec2) {
; CHECK-LABEL: interleave3_v24i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #48
-; CHECK-NEXT: .cfi_def_cfa_offset 48
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x9]
-; CHECK-NEXT: ldp q1, q0, [sp, #16]
-; CHECK-NEXT: ldr q2, [sp]
-; CHECK-NEXT: stp q1, q0, [x8, #16]
-; CHECK-NEXT: str q2, [x8]
-; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x8]
; CHECK-NEXT: ret
%retval = call <24 x i16> @llvm.vector.interleave3.v24i16(<8 x i16> %vec0, <8 x i16> %vec1, <8 x i16> %vec2)
ret <24 x i16> %retval
@@ -495,18 +533,10 @@ define <24 x i16> @interleave3_v24i16(<8 x i16> %vec0, <8 x i16> %vec1, <8 x i16
define <24 x half> @interleave3_v24f16(<8 x half> %vec0, <8 x half> %vec1, <8 x half> %vec2) {
; CHECK-LABEL: interleave3_v24f16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #48
-; CHECK-NEXT: .cfi_def_cfa_offset 48
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x9]
-; CHECK-NEXT: ldp q1, q0, [sp, #16]
-; CHECK-NEXT: ldr q2, [sp]
-; CHECK-NEXT: stp q1, q0, [x8, #16]
-; CHECK-NEXT: str q2, [x8]
-; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x8]
; CHECK-NEXT: ret
%retval = call <24 x half> @llvm.vector.interleave3.v24f16(<8 x half> %vec0, <8 x half> %vec1, <8 x half> %vec2)
ret <24 x half> %retval
@@ -515,18 +545,10 @@ define <24 x half> @interleave3_v24f16(<8 x half> %vec0, <8 x half> %vec1, <8 x
define <24 x bfloat> @interleave3_v24bf16(<8 x bfloat> %vec0, <8 x bfloat> %vec1, <8 x bfloat> %vec2) {
; CHECK-LABEL: interleave3_v24bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #48
-; CHECK-NEXT: .cfi_def_cfa_offset 48
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x9]
-; CHECK-NEXT: ldp q1, q0, [sp, #16]
-; CHECK-NEXT: ldr q2, [sp]
-; CHECK-NEXT: stp q1, q0, [x8, #16]
-; CHECK-NEXT: str q2, [x8]
-; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: st3 { v0.8h, v1.8h, v2.8h }, [x8]
; CHECK-NEXT: ret
%retval = call <24 x bfloat> @llvm.vector.interleave3.v24bf16(<8 x bfloat> %vec0, <8 x bfloat> %vec1, <8 x bfloat> %vec2)
ret <24 x bfloat> %retval
@@ -535,18 +557,10 @@ define <24 x bfloat> @interleave3_v24bf16(<8 x bfloat> %vec0, <8 x bfloat> %vec1
define <48 x i8> @interleave3_v48i8(<16 x i8> %vec0, <16 x i8> %vec1, <16 x i8> %vec2) {
; CHECK-LABEL: interleave3_v48i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #48
-; CHECK-NEXT: .cfi_def_cfa_offset 48
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: mov x9, sp
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; CHECK-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x9]
-; CHECK-NEXT: ldp q1, q0, [sp, #16]
-; CHECK-NEXT: ldr q2, [sp]
-; CHECK-NEXT: stp q1, q0, [x8, #16]
-; CHECK-NEXT: str q2, [x8]
-; CHECK-NEXT: add sp, sp, #48
+; CHECK-NEXT: st3 { v0.16b, v1.16b, v2.16b }, [x8]
; CHECK-NEXT: ret
%retval = call <48 x i8> @llvm.vector.interleave3.v48i8(<16 x i8> %vec0, <16 x i8> %vec1, <16 x i8> %vec2)
ret <48 x i8> %retval
@@ -555,19 +569,10 @@ define <48 x i8> @interleave3_v48i8(<16 x i8> %vec0, <16 x i8> %vec1, <16 x i8>
define <12 x i16> @interleave3_v12i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2) {
; CHECK-LABEL: interleave3_v12i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #32
-; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT: add x9, sp, #8
; CHECK-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT: st3 { v0.4h, v1.4h, v2.4h }, [x9]
-; CHECK-NEXT: ldp d1, d0, [sp, #8]
-; CHECK-NEXT: mov v1.d[1], v0.d[0]
-; CHECK-NEXT: ldr d0, [sp, #24]
-; CHECK-NEXT: str d0, [x8, #16]
-; CHECK-NEXT: str q1, [x8]
-; CHECK-NEXT: add sp, sp, #32
+; CHECK-NEXT: st3 { v0.4h, v1.4h, v2.4h }, [x8]
; CHECK-NEXT: ret
%retval = call <12 x i16> @llvm.vector.interleave3.v12i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2)
ret <12 x i16> %retval
@@ -576,19 +581,10 @@ define <12 x i16> @interleave3_v12i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16
define <24 x i8> @interleave3_v24i8(<8 x i8> %vec0, <8 x i8> %vec1, <8 x i8> %vec2) {
; CHECK-LABEL: interleave3_v24i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #32
-; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: // kill: def $d2 killed $d2 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT: add x9, sp, #8
; CHECK-NEXT: // kill: def $d1 killed $d1 killed $d0_d1_d2 def $d0_d1_d2
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $d0_d1_d2 def $d0_d1_d2
-; CHECK-NEXT: st3 { v0.8b, v1.8b, v2.8b }, [x9]
-; CHECK-NEXT: ldp d1, d0, [sp, #8]
-; CHECK-NEXT: mov v1.d[1], v0.d[0]
-; CHECK-NEXT: ldr d0, [sp, #24]
-; CHECK-NEXT: str d0, [x8, #16]
-; CHECK-NEXT: str q1, [x8]
-; CHECK-NEXT: add sp, sp, #32
+; CHECK-NEXT: st3 { v0.8b, v1.8b, v2.8b }, [x8]
; CHECK-NEXT: ret
%retval = call <24 x i8> @llvm.vector.interleave3.v24i8(<8 x i8> %vec0, <8 x i8> %vec1, <8 x i8> %vec2)
ret <24 x i8> %retval
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index d3cd9bf08cc0a..41c8f7abcdabc 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false -mattr=+sve < %s | FileCheck %s
define void @foo_st2_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_nxv16i8:
@@ -159,10 +160,10 @@ define void @foo_st2_nxv16i8_all_false_mask(<vscale x 16 x i8> %val1, <vscale x
define void @foo_st2_nxv16i8_all_true_mask(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_nxv16i8_all_true_mask:
; CHECK: // %bb.0:
-; CHECK-NEXT: zip2 z2.b, z0.b, z1.b
-; CHECK-NEXT: zip1 z0.b, z0.b, z1.b
-; CHECK-NEXT: str z2, [x0, #1, mul vl]
-; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
; CHECK-NEXT: ret
%interleaved.value = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2)
call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> splat(i1 1))
diff --git a/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll b/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
index c79dc0624b5c3..a7ad6906ed7e2 100644
--- a/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/AArch64/vector-deinterleave-load.ll
@@ -3,31 +3,17 @@
; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,IA-DISABLE
define void @aarch64_vector_deinterleave_idx_ld2(ptr %ptr, i64 %idx) {
-; IA-ENABLE-LABEL: aarch64_vector_deinterleave_idx_ld2:
-; IA-ENABLE: // %bb.0: // %entry
-; IA-ENABLE-NEXT: lsl x8, x1, #2
-; IA-ENABLE-NEXT: and x9, x8, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: add x8, x8, #16
-; IA-ENABLE-NEXT: add x10, x0, x9
-; IA-ENABLE-NEXT: and x8, x8, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: ld2 { v0.4s, v1.4s }, [x10]
-; IA-ENABLE-NEXT: str q0, [x0, x9]
-; IA-ENABLE-NEXT: str q1, [x0, x8]
-; IA-ENABLE-NEXT: ret
-;
-; IA-DISABLE-LABEL: aarch64_vector_deinterleave_idx_ld2:
-; IA-DISABLE: // %bb.0: // %entry
-; IA-DISABLE-NEXT: lsl x8, x1, #2
-; IA-DISABLE-NEXT: and x9, x8, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: add x8, x8, #16
-; IA-DISABLE-NEXT: add x9, x0, x9
-; IA-DISABLE-NEXT: and x8, x8, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: ldp q1, q0, [x9]
-; IA-DISABLE-NEXT: uzp1 v2.4s, v1.4s, v0.4s
-; IA-DISABLE-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; IA-DISABLE-NEXT: str q2, [x9]
-; IA-DISABLE-NEXT: str q0, [x0, x8]
-; IA-DISABLE-NEXT: ret
+; CHECK-LABEL: aarch64_vector_deinterleave_idx_ld2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsl x8, x1, #2
+; CHECK-NEXT: and x9, x8, #0xfffffffffffffff0
+; CHECK-NEXT: add x8, x8, #16
+; CHECK-NEXT: add x10, x0, x9
+; CHECK-NEXT: and x8, x8, #0xfffffffffffffff0
+; CHECK-NEXT: ld2 { v0.4s, v1.4s }, [x10]
+; CHECK-NEXT: str q0, [x0, x9]
+; CHECK-NEXT: str q1, [x0, x8]
+; CHECK-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%a1 = add i64 %idx, 4
@@ -47,43 +33,20 @@ entry:
}
define void @aarch64_vector_deinterleave_idx_ld3(ptr %ptr, i64 %idx) {
-; IA-ENABLE-LABEL: aarch64_vector_deinterleave_idx_ld3:
-; IA-ENABLE: // %bb.0: // %entry
-; IA-ENABLE-NEXT: lsl x8, x1, #2
-; IA-ENABLE-NEXT: and x9, x8, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: add x10, x0, x9
-; IA-ENABLE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x10]
-; IA-ENABLE-NEXT: add x10, x8, #16
-; IA-ENABLE-NEXT: add x8, x8, #32
-; IA-ENABLE-NEXT: and x10, x10, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: and x8, x8, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: str q0, [x0, x9]
-; IA-ENABLE-NEXT: str q1, [x0, x10]
-; IA-ENABLE-NEXT: str q2, [x0, x8]
-; IA-ENABLE-NEXT: ret
-;
-; IA-DISABLE-LABEL: aarch64_vector_deinterleave_idx_ld3:
-; IA-DISABLE: // %bb.0: // %entry
-; IA-DISABLE-NEXT: sub sp, sp, #48
-; IA-DISABLE-NEXT: .cfi_def_cfa_offset 48
-; IA-DISABLE-NEXT: lsl x9, x1, #2
-; IA-DISABLE-NEXT: mov x8, sp
-; IA-DISABLE-NEXT: and x10, x9, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: add x10, x0, x10
-; IA-DISABLE-NEXT: ldp q1, q0, [x10, #16]
-; IA-DISABLE-NEXT: ldr q2, [x10]
-; IA-DISABLE-NEXT: str q2, [sp]
-; IA-DISABLE-NEXT: stp q1, q0, [sp, #16]
-; IA-DISABLE-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x8]
-; IA-DISABLE-NEXT: add x8, x9, #16
-; IA-DISABLE-NEXT: add x9, x9, #32
-; IA-DISABLE-NEXT: and x8, x8, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: and x9, x9, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: str q0, [x10]
-; IA-DISABLE-NEXT: str q1, [x0, x8]
-; IA-DISABLE-NEXT: str q2, [x0, x9]
-; IA-DISABLE-NEXT: add sp, sp, #48
-; IA-DISABLE-NEXT: ret
+; CHECK-LABEL: aarch64_vector_deinterleave_idx_ld3:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsl x8, x1, #2
+; CHECK-NEXT: and x9, x8, #0xfffffffffffffff0
+; CHECK-NEXT: add x10, x0, x9
+; CHECK-NEXT: ld3 { v0.4s, v1.4s, v2.4s }, [x10]
+; CHECK-NEXT: add x10, x8, #16
+; CHECK-NEXT: add x8, x8, #32
+; CHECK-NEXT: and x10, x10, #0xfffffffffffffff0
+; CHECK-NEXT: and x8, x8, #0xfffffffffffffff0
+; CHECK-NEXT: str q0, [x0, x9]
+; CHECK-NEXT: str q1, [x0, x10]
+; CHECK-NEXT: str q2, [x0, x8]
+; CHECK-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%a1 = add i64 %idx, 4
@@ -108,52 +71,24 @@ entry:
}
define void @aarch64_vector_deinterleave_idx_ld4(ptr %ptr, i64 %idx) {
-; IA-ENABLE-LABEL: aarch64_vector_deinterleave_idx_ld4:
-; IA-ENABLE: // %bb.0: // %entry
-; IA-ENABLE-NEXT: lsl x8, x1, #2
-; IA-ENABLE-NEXT: add x9, x8, #64
-; IA-ENABLE-NEXT: add x11, x8, #96
-; IA-ENABLE-NEXT: and x9, x9, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: add x10, x0, x9
-; IA-ENABLE-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x10]
-; IA-ENABLE-NEXT: add x10, x8, #80
-; IA-ENABLE-NEXT: add x8, x8, #112
-; IA-ENABLE-NEXT: and x10, x10, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: and x8, x8, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: str q0, [x0, x9]
-; IA-ENABLE-NEXT: and x9, x11, #0xfffffffffffffff0
-; IA-ENABLE-NEXT: str q1, [x0, x10]
-; IA-ENABLE-NEXT: str q2, [x0, x9]
-; IA-ENABLE-NEXT: str q3, [x0, x8]
-; IA-ENABLE-NEXT: ret
-;
-; IA-DISABLE-LABEL: aarch64_vector_deinterleave_idx_ld4:
-; IA-DISABLE: // %bb.0: // %entry
-; IA-DISABLE-NEXT: lsl x8, x1, #2
-; IA-DISABLE-NEXT: add x9, x8, #64
-; IA-DISABLE-NEXT: add x10, x8, #80
-; IA-DISABLE-NEXT: add x11, x8, #96
-; IA-DISABLE-NEXT: and x9, x9, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: add x8, x8, #112
-; IA-DISABLE-NEXT: and x10, x10, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: add x9, x0, x9
-; IA-DISABLE-NEXT: and x8, x8, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: ldp q1, q0, [x9]
-; IA-DISABLE-NEXT: ldp q3, q2, [x9, #32]
-; IA-DISABLE-NEXT: uzp1 v5.4s, v1.4s, v0.4s
-; IA-DISABLE-NEXT: uzp2 v0.4s, v1.4s, v0.4s
-; IA-DISABLE-NEXT: uzp1 v4.4s, v3.4s, v2.4s
-; IA-DISABLE-NEXT: uzp2 v2.4s, v3.4s, v2.4s
-; IA-DISABLE-NEXT: uzp1 v1.4s, v5.4s, v4.4s
-; IA-DISABLE-NEXT: uzp1 v3.4s, v0.4s, v2.4s
-; IA-DISABLE-NEXT: uzp2 v4.4s, v5.4s, v4.4s
-; IA-DISABLE-NEXT: uzp2 v0.4s, v0.4s, v2.4s
-; IA-DISABLE-NEXT: str q1, [x9]
-; IA-DISABLE-NEXT: and x9, x11, #0xfffffffffffffff0
-; IA-DISABLE-NEXT: str q3, [x0, x10]
-; IA-DISABLE-NEXT: str q4, [x0, x9]
-; IA-DISABLE-NEXT: str q0, [x0, x8]
-; IA-DISABLE-NEXT: ret
+; CHECK-LABEL: aarch64_vector_deinterleave_idx_ld4:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsl x8, x1, #2
+; CHECK-NEXT: add x9, x8, #64
+; CHECK-NEXT: add x11, x8, #96
+; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0
+; CHECK-NEXT: add x10, x0, x9
+; CHECK-NEXT: ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x10]
+; CHECK-NEXT: add x10, x8, #80
+; CHECK-NEXT: add x8, x8, #112
+; CHECK-NEXT: and x10, x10, #0xfffffffffffffff0
+; CHECK-NEXT: and x8, x8, #0xfffffffffffffff0
+; CHECK-NEXT: str q0, [x0, x9]
+; CHECK-NEXT: and x9, x11, #0xfffffffffffffff0
+; CHECK-NEXT: str q1, [x0, x10]
+; CHECK-NEXT: str q2, [x0, x9]
+; CHECK-NEXT: str q3, [x0, x8]
+; CHECK-NEXT: ret
entry:
%a1 = add i64 %idx, 16
%idx1 = lshr i64 %a1, 2
@@ -183,4 +118,5 @@ entry:
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
+; IA-DISABLE: {{.*}}
+; IA-ENABLE: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
index c5edb6cf82984..a7c16fda11478 100644
--- a/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/AArch64/vector-interleave-store.ll
@@ -2,23 +2,14 @@
; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,IA-ENABLE
; RUN: llc -mtriple=aarch64-linux-gnu -lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,IA-DISABLE
define void @aarch64_vector_interleave_idx_st2(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1) {
-; IA-ENABLE-LABEL: aarch64_vector_interleave_idx_st2:
-; IA-ENABLE: // %bb.0: // %entry
-; IA-ENABLE-NEXT: lsr x8, x1, #2
-; IA-ENABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
-; IA-ENABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
-; IA-ENABLE-NEXT: add x8, x0, x8, lsl #4
-; IA-ENABLE-NEXT: st2 { v0.4s, v1.4s }, [x8]
-; IA-ENABLE-NEXT: ret
-;
-; IA-DISABLE-LABEL: aarch64_vector_interleave_idx_st2:
-; IA-DISABLE: // %bb.0: // %entry
-; IA-DISABLE-NEXT: lsr x8, x1, #2
-; IA-DISABLE-NEXT: zip2 v2.4s, v0.4s, v1.4s
-; IA-DISABLE-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; IA-DISABLE-NEXT: add x8, x0, x8, lsl #4
-; IA-DISABLE-NEXT: stp q0, q2, [x8]
-; IA-DISABLE-NEXT: ret
+; CHECK-LABEL: aarch64_vector_interleave_idx_st2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsr x8, x1, #2
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1
+; CHECK-NEXT: add x8, x0, x8, lsl #4
+; CHECK-NEXT: st2 { v0.4s, v1.4s }, [x8]
+; CHECK-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
@@ -29,33 +20,15 @@ entry:
}
define void @aarch64_vector_interleave_idx_st3(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1, <4 x float> %v2) {
-; IA-ENABLE-LABEL: aarch64_vector_interleave_idx_st3:
-; IA-ENABLE: // %bb.0: // %entry
-; IA-ENABLE-NEXT: lsr x8, x1, #2
-; IA-ENABLE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; IA-ENABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; IA-ENABLE-NEXT: add x8, x0, x8, lsl #4
-; IA-ENABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; IA-ENABLE-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
-; IA-ENABLE-NEXT: ret
-;
-; IA-DISABLE-LABEL: aarch64_vector_interleave_idx_st3:
-; IA-DISABLE: // %bb.0: // %entry
-; IA-DISABLE-NEXT: sub sp, sp, #48
-; IA-DISABLE-NEXT: .cfi_def_cfa_offset 48
-; IA-DISABLE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
-; IA-DISABLE-NEXT: mov x8, sp
-; IA-DISABLE-NEXT: lsr x9, x1, #2
-; IA-DISABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
-; IA-DISABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
-; IA-DISABLE-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
-; IA-DISABLE-NEXT: add x8, x0, x9, lsl #4
-; IA-DISABLE-NEXT: ldp q1, q0, [sp, #16]
-; IA-DISABLE-NEXT: ldr q2, [sp]
-; IA-DISABLE-NEXT: stp q1, q0, [x8, #16]
-; IA-DISABLE-NEXT: str q2, [x8]
-; IA-DISABLE-NEXT: add sp, sp, #48
-; IA-DISABLE-NEXT: ret
+; CHECK-LABEL: aarch64_vector_interleave_idx_st3:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsr x8, x1, #2
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-NEXT: add x8, x0, x8, lsl #4
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2 def $q0_q1_q2
+; CHECK-NEXT: st3 { v0.4s, v1.4s, v2.4s }, [x8]
+; CHECK-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
@@ -66,32 +39,16 @@ entry:
}
define void @aarch64_vector_interleave_idx_st4(ptr %ptr, i64 %idx, <4 x float> %v0, <4 x float> %v1, <4 x float> %v2, <4 x float> %v3) {
-; IA-ENABLE-LABEL: aarch64_vector_interleave_idx_st4:
-; IA-ENABLE: // %bb.0: // %entry
-; IA-ENABLE-NEXT: lsr x8, x1, #2
-; IA-ENABLE-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; IA-ENABLE-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; IA-ENABLE-NEXT: add x8, x0, x8, lsl #4
-; IA-ENABLE-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; IA-ENABLE-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
-; IA-ENABLE-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
-; IA-ENABLE-NEXT: ret
-;
-; IA-DISABLE-LABEL: aarch64_vector_interleave_idx_st4:
-; IA-DISABLE: // %bb.0: // %entry
-; IA-DISABLE-NEXT: zip2 v4.4s, v1.4s, v3.4s
-; IA-DISABLE-NEXT: zip2 v5.4s, v0.4s, v2.4s
-; IA-DISABLE-NEXT: lsr x8, x1, #2
-; IA-DISABLE-NEXT: zip1 v1.4s, v1.4s, v3.4s
-; IA-DISABLE-NEXT: zip1 v0.4s, v0.4s, v2.4s
-; IA-DISABLE-NEXT: add x8, x0, x8, lsl #4
-; IA-DISABLE-NEXT: zip2 v2.4s, v5.4s, v4.4s
-; IA-DISABLE-NEXT: zip1 v3.4s, v5.4s, v4.4s
-; IA-DISABLE-NEXT: zip2 v4.4s, v0.4s, v1.4s
-; IA-DISABLE-NEXT: zip1 v0.4s, v0.4s, v1.4s
-; IA-DISABLE-NEXT: stp q3, q2, [x8, #32]
-; IA-DISABLE-NEXT: stp q0, q4, [x8]
-; IA-DISABLE-NEXT: ret
+; CHECK-LABEL: aarch64_vector_interleave_idx_st4:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: lsr x8, x1, #2
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT: add x8, x0, x8, lsl #4
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3
+; CHECK-NEXT: st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [x8]
+; CHECK-NEXT: ret
entry:
%idx1 = lshr i64 %idx, 2
%gep1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 %idx1
@@ -101,4 +58,5 @@ entry:
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
+; IA-DISABLE: {{.*}}
+; IA-ENABLE: {{.*}}
More information about the llvm-commits
mailing list