[llvm] [AArch64] Lower fixed-length interleaved stores with SVE (PR #213692)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 07:52:34 PDT 2026
https://github.com/kamleshbhalui updated https://github.com/llvm/llvm-project/pull/213692
>From 03538fdb6c91ac67cba4e85534c39f3fea235150 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Mon, 3 Aug 2026 12:43:51 +0000
Subject: [PATCH 1/3] [AArch64] Lower fixed-length interleaved stores with SVE
Allow uses of SVE instruction for fixed length vector
interleave intrinsic that can not be lowered through NEON.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 45 +++++-
.../AArch64/fixed-length-sve-interleave.ll | 153 ++++++++++++++++++
2 files changed, 192 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 51be0e66b19b0..97e166b782168 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3715,6 +3715,8 @@ static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
static SDValue convertFixedMaskToScalableVector(SDValue Mask,
SelectionDAG &DAG);
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
+static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
+ EVT VT);
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
EVT VT);
static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG);
@@ -27775,12 +27777,8 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
return SDValue();
bool IsScalable = SubVecTy.isScalableVector();
unsigned SubBits = SubVecTy.getSizeInBits().getKnownMinValue();
- if (IsScalable) {
- if (SubBits != 128)
- return SDValue();
- } else if (SubBits != 64 && SubBits != 128) {
+ if (IsScalable && SubBits != 128)
return SDValue();
- }
auto *MemN = cast<MemSDNode>(N);
if (IsScalable) {
@@ -27805,8 +27803,12 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
DAG.getVTList(MVT::Other), Ops,
MemN->getMemoryVT(), MemN->getMemOperand());
- } else {
+ }
+ const AArch64Subtarget &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
+ // Fixed length vector using NEON
+ if (!IsMasked && (SubBits == 64 || SubBits == 128) &&
+ Subtarget.isNeonAvailable()) {
static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2,
Intrinsic::aarch64_neon_st3,
Intrinsic::aarch64_neon_st4};
@@ -27820,6 +27822,37 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
DAG.getVTList(MVT::Other), Ops,
MemN->getMemoryVT(), MemN->getMemOperand());
}
+
+ // Fixed-length vectors using SVE
+ if (!Subtarget.isSVEorStreamingSVEAvailable())
+ return SDValue();
+
+ EVT ContainerVT = getContainerForFixedLengthVector(DAG, SubVecTy);
+ SDValue Pred;
+ if (IsMasked) {
+ Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
+ if (!Pred)
+ return SDValue();
+ EVT MaskVT = SubVecTy.changeTypeToInteger();
+ // Widen the i1 mask
+ if (Pred.getValueType() != MaskVT)
+ Pred = DAG.getNode(ISD::SIGN_EXTEND, DL, MaskVT, Pred);
+ Pred = convertFixedMaskToScalableVector(Pred, DAG);
+ } else {
+ Pred = getPredicateForFixedLengthVector(DAG, DL, SubVecTy);
+ }
+
+ static constexpr Intrinsic::ID SVEStores[] = {Intrinsic::aarch64_sve_st2,
+ Intrinsic::aarch64_sve_st3,
+ Intrinsic::aarch64_sve_st4};
+ SmallVector<SDValue, 8> Ops;
+ Ops.append({Chain, DAG.getConstant(SVEStores[NumParts - 2], DL, MVT::i32)});
+ for (SDValue V : ValueInterleaveOps)
+ Ops.push_back(convertToScalableVector(DAG, ContainerVT, V));
+ Ops.append({Pred, BasePtr});
+ return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
+ DAG.getVTList(MVT::Other), Ops,
+ MemN->getMemoryVT(), MemN->getMemOperand());
}
static SDValue performMSTORECombine(SDNode *N,
diff --git a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
new file mode 100644
index 0000000000000..e3ef467284269
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+
+define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
+; CHECK-LABEL: store_factor2_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: adrp x8, .LCPI0_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI0_0
+; CHECK-NEXT: ldr z4, [x8]
+; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
+; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
+; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
+; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <32 x i16> @llvm.vector.interleave2.v32i16(<16 x i16> %v0, <16 x i16> %v1)
+ store <32 x i16> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
+; CHECK-LABEL: store_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
+; CHECK-LABEL: store_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ store <16 x i64> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @masked_store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: // kill: def $d6 killed $d6 def $z6
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: uunpklo z0.h, z6.b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: lsl z0.s, z0.s, #31
+; CHECK-NEXT: asr z0.s, z0.s, #31
+; CHECK-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ %interleaved.mask = call <24 x i1> @llvm.vector.interleave3.v24i1(<8 x i1> %mask, <8 x i1> %mask, <8 x i1> %mask)
+ call void @llvm.masked.store.v24i32.p0(<24 x i32> %interleaved.vec, ptr %ptr, i32 4, <24 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: ldr d2, [sp]
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: uunpklo z0.d, z2.s
+; CHECK-NEXT: lsl z0.d, z0.d, #63
+; CHECK-NEXT: asr z0.d, z0.d, #63
+; CHECK-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ %interleaved.mask = call <16 x i1> @llvm.vector.interleave4.v16i1(<4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v16i64.p0(<16 x i64> %interleaved.vec, ptr %ptr, i32 8, <16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor3_128bit_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor3_128bit_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ %interleaved.mask = call <12 x i1> @llvm.vector.interleave3.v12i1(<4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v12i32.p0(<12 x i32> %interleaved.vec, ptr %ptr, i32 4, <12 x i1> %interleaved.mask)
+ ret void
+}
+
+attributes #0 = { vscale_range(2,2) "target-features"="+sve" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-IADISABLED: {{.*}}
+; CHECK-IAENABLED: {{.*}}
>From e2c79dcf4bcd30a2f13ddaf75d6d4e26be5c23b2 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Mon, 3 Aug 2026 16:11:11 +0000
Subject: [PATCH 2/3] fixup added sme vla test
---
...-streaming-mode-fixed-length-interleave.ll | 286 ++++++++++++++++++
1 file changed, 286 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
new file mode 100644
index 0000000000000..e5f70cead7001
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK-COMPAT,CHECK-COMPAT-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK-COMPAT,CHECK-COMPAT-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK-STREAMING,CHECK-STREAMING-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK-STREAMING,CHECK-STREAMING-IADISABLED
+
+define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
+; CHECK-COMPAT-LABEL: store_factor3:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-COMPAT-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-COMPAT-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl8
+; CHECK-COMPAT-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor3:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: splice z4.s, p0, { z4.s, z5.s }
+; CHECK-STREAMING-NEXT: splice z3.s, p0, { z2.s, z3.s }
+; CHECK-STREAMING-NEXT: splice z2.s, p0, { z0.s, z1.s }
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl8
+; CHECK-STREAMING-NEXT: st3w { z2.s - z4.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <24 x i32> @llvm.vector.interleave3.v24i32(
+ <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleaved, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
+; CHECK-COMPAT-LABEL: store_factor4:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z19.d, z6.d
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
+; CHECK-COMPAT-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-COMPAT-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-COMPAT-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-COMPAT-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl4
+; CHECK-COMPAT-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor4:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
+; CHECK-STREAMING-NEXT: splice z6.d, p0, { z6.d, z7.d }
+; CHECK-STREAMING-NEXT: splice z5.d, p0, { z4.d, z5.d }
+; CHECK-STREAMING-NEXT: splice z4.d, p0, { z2.d, z3.d }
+; CHECK-STREAMING-NEXT: splice z3.d, p0, { z0.d, z1.d }
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl4
+; CHECK-STREAMING-NEXT: st4d { z3.d - z6.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <16 x i64> @llvm.vector.interleave4.v16i64(
+ <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ store <16 x i64> %interleaved, ptr %ptr, align 8
+ ret void
+}
+
+
+define void @store_factor2_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) #0 {
+; CHECK-COMPAT-LABEL: store_factor2_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: adrp x8, .LCPI2_0
+; CHECK-COMPAT-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-COMPAT-NEXT: ptrue p1.s, vl8
+; CHECK-COMPAT-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-COMPAT-NEXT: ld1w { z1.s }, p1/z, [x8]
+; CHECK-COMPAT-NEXT: tbl z0.s, { z0.s }, z1.s
+; CHECK-COMPAT-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor2_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: adrp x8, .LCPI2_0
+; CHECK-STREAMING-NEXT: add x8, x8, :lo12:.LCPI2_0
+; CHECK-STREAMING-NEXT: ptrue p1.s, vl8
+; CHECK-STREAMING-NEXT: splice z0.s, p0, { z0.s, z1.s }
+; CHECK-STREAMING-NEXT: ld1w { z1.s }, p1/z, [x8]
+; CHECK-STREAMING-NEXT: tbl z0.s, { z0.s }, z1.s
+; CHECK-STREAMING-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(
+ <4 x i32> %v0, <4 x i32> %v1)
+ store <8 x i32> %interleaved, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) #0 {
+; CHECK-COMPAT-LABEL: store_factor3_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: st3w { z0.s - z2.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor3_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: st3w { z0.s - z2.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <12 x i32> @llvm.vector.interleave3.v12i32(
+ <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ store <12 x i32> %interleaved, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_128bit(ptr %ptr, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3) #0 {
+; CHECK-COMPAT-LABEL: store_factor4_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
+; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor4_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
+; CHECK-STREAMING-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <8 x i64> @llvm.vector.interleave4.v8i64(
+ <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3)
+ store <8 x i64> %interleaved, ptr %ptr, align 8
+ ret void
+}
+
+define void @masked_store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) #0 {
+; CHECK-COMPAT-LABEL: masked_store_factor3:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: // kill: def $d6 killed $d6 def $z6
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: ptrue p1.s, vl8
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: uunpklo z0.h, z6.b
+; CHECK-COMPAT-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-COMPAT-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-COMPAT-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-COMPAT-NEXT: uunpklo z0.s, z0.h
+; CHECK-COMPAT-NEXT: lsl z0.s, z0.s, #31
+; CHECK-COMPAT-NEXT: asr z0.s, z0.s, #31
+; CHECK-COMPAT-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-COMPAT-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor3:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: uunpklo z6.h, z6.b
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: ptrue p1.s, vl8
+; CHECK-STREAMING-NEXT: splice z4.s, p0, { z4.s, z5.s }
+; CHECK-STREAMING-NEXT: splice z3.s, p0, { z2.s, z3.s }
+; CHECK-STREAMING-NEXT: splice z2.s, p0, { z0.s, z1.s }
+; CHECK-STREAMING-NEXT: uunpklo z6.s, z6.h
+; CHECK-STREAMING-NEXT: lsl z6.s, z6.s, #31
+; CHECK-STREAMING-NEXT: asr z6.s, z6.s, #31
+; CHECK-STREAMING-NEXT: cmpne p0.s, p1/z, z6.s, #0
+; CHECK-STREAMING-NEXT: st3w { z2.s - z4.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <24 x i32> @llvm.vector.interleave3.v24i32(
+ <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ %interleaved.mask = call <24 x i1> @llvm.vector.interleave3.v24i1(
+ <8 x i1> %mask, <8 x i1> %mask, <8 x i1> %mask)
+ call void @llvm.masked.store.v24i32.p0(<24 x i32> %interleaved, ptr %ptr,
+ i32 4, <24 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) #0 {
+; CHECK-COMPAT-LABEL: masked_store_factor4:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: mov z19.d, z6.d
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
+; CHECK-COMPAT-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-COMPAT-NEXT: mov z18.d, z4.d
+; CHECK-COMPAT-NEXT: ptrue p1.d, vl4
+; CHECK-COMPAT-NEXT: mov z17.d, z2.d
+; CHECK-COMPAT-NEXT: ldr d2, [sp]
+; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: uunpklo z2.s, z2.h
+; CHECK-COMPAT-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-COMPAT-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-COMPAT-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-COMPAT-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-COMPAT-NEXT: uunpklo z0.d, z2.s
+; CHECK-COMPAT-NEXT: lsl z0.d, z0.d, #63
+; CHECK-COMPAT-NEXT: asr z0.d, z0.d, #63
+; CHECK-COMPAT-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-COMPAT-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor4:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ldr d16, [sp]
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
+; CHECK-STREAMING-NEXT: ptrue p1.d, vl4
+; CHECK-STREAMING-NEXT: uunpklo z16.s, z16.h
+; CHECK-STREAMING-NEXT: splice z6.d, p0, { z6.d, z7.d }
+; CHECK-STREAMING-NEXT: splice z5.d, p0, { z4.d, z5.d }
+; CHECK-STREAMING-NEXT: splice z4.d, p0, { z2.d, z3.d }
+; CHECK-STREAMING-NEXT: splice z3.d, p0, { z0.d, z1.d }
+; CHECK-STREAMING-NEXT: uunpklo z16.d, z16.s
+; CHECK-STREAMING-NEXT: lsl z16.d, z16.d, #63
+; CHECK-STREAMING-NEXT: asr z7.d, z16.d, #63
+; CHECK-STREAMING-NEXT: cmpne p0.d, p1/z, z7.d, #0
+; CHECK-STREAMING-NEXT: st4d { z3.d - z6.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <16 x i64> @llvm.vector.interleave4.v16i64(
+ <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ %interleaved.mask = call <16 x i1> @llvm.vector.interleave4.v16i1(
+ <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v16i64.p0(<16 x i64> %interleaved, ptr %ptr,
+ i32 8, <16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) #0 {
+; CHECK-COMPAT-LABEL: masked_store_factor3_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $d3 killed $d3 def $z3
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: uunpklo z3.s, z3.h
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: lsl z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: asr z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-COMPAT-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor3_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: uunpklo z3.s, z3.h
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: lsl z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: asr z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-STREAMING-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <12 x i32> @llvm.vector.interleave3.v12i32(
+ <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ %interleaved.mask = call <12 x i1> @llvm.vector.interleave3.v12i1(
+ <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v12i32.p0(<12 x i32> %interleaved, ptr %ptr,
+ i32 4, <12 x i1> %interleaved.mask)
+ ret void
+}
+
+attributes #0 = { vscale_range(2,16) }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-COMPAT-IADISABLED: {{.*}}
+; CHECK-COMPAT-IAENABLED: {{.*}}
+; CHECK-STREAMING-IADISABLED: {{.*}}
+; CHECK-STREAMING-IAENABLED: {{.*}}
>From 70350e47665ed7f48260b92fe263be494c585ed2 Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Wed, 5 Aug 2026 14:51:38 +0000
Subject: [PATCH 3/3] fixup udpated tests keep neon sizes only for sme
---
.../AArch64/fixed-length-sve-interleave.ll | 17 +-
...-streaming-mode-fixed-length-interleave.ll | 432 +++++++++++-------
2 files changed, 284 insertions(+), 165 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
index e3ef467284269..1278606f5b63f 100644
--- a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
-; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
-define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
+define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) vscale_range(2,2) {
; CHECK-LABEL: store_factor2_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl8
@@ -25,7 +25,7 @@ define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #
ret void
}
-define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
+define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) vscale_range(2,2) {
; CHECK-LABEL: store_factor3_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov v18.16b, v4.16b
@@ -46,7 +46,7 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
ret void
}
-define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
+define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) vscale_range(2,2) {
; CHECK-LABEL: store_factor4_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov v19.16b, v6.16b
@@ -70,7 +70,7 @@ define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4
ret void
}
-define void @masked_store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) #0 {
+define void @masked_store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) vscale_range(2,2) {
; CHECK-LABEL: masked_store_factor3_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov v18.16b, v4.16b
@@ -98,7 +98,7 @@ define void @masked_store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %
ret void
}
-define void @masked_store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) #0 {
+define void @masked_store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) vscale_range(2,2) {
; CHECK-LABEL: masked_store_factor4_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov v19.16b, v6.16b
@@ -129,7 +129,7 @@ define void @masked_store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %
ret void
}
-define void @masked_store_factor3_128bit_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) #0 {
+define void @masked_store_factor3_128bit_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) vscale_range(2,2) {
; CHECK-LABEL: masked_store_factor3_128bit_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: ushll v3.4s, v3.4h, #0
@@ -147,7 +147,6 @@ define void @masked_store_factor3_128bit_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x
ret void
}
-attributes #0 = { vscale_range(2,2) "target-features"="+sve" }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK-IADISABLED: {{.*}}
; CHECK-IAENABLED: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
index e5f70cead7001..d87b6b0404874 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-interleave.ll
@@ -4,108 +4,91 @@
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK-STREAMING,CHECK-STREAMING-IAENABLED
; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK-STREAMING,CHECK-STREAMING-IADISABLED
-define void @store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
-; CHECK-COMPAT-LABEL: store_factor3:
+
+define void @store_factor2_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1) {
+; CHECK-COMPAT-LABEL: store_factor2_64bit:
; CHECK-COMPAT: // %bb.0:
-; CHECK-COMPAT-NEXT: mov z18.d, z4.d
-; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
-; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-COMPAT-NEXT: mov z17.d, z2.d
-; CHECK-COMPAT-NEXT: mov z16.d, z0.d
-; CHECK-COMPAT-NEXT: splice z18.s, p0, z18.s, z5.s
-; CHECK-COMPAT-NEXT: splice z17.s, p0, z17.s, z3.s
-; CHECK-COMPAT-NEXT: splice z16.s, p0, z16.s, z1.s
-; CHECK-COMPAT-NEXT: ptrue p0.s, vl8
-; CHECK-COMPAT-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl2
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-COMPAT-NEXT: adrp x8, .LCPI0_0
+; CHECK-COMPAT-NEXT: splice z0.s, p0, z0.s, z1.s
+; CHECK-COMPAT-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-COMPAT-NEXT: tbl z0.s, { z0.s }, z1.s
+; CHECK-COMPAT-NEXT: str q0, [x0]
; CHECK-COMPAT-NEXT: ret
;
-; CHECK-STREAMING-LABEL: store_factor3:
+; CHECK-STREAMING-LABEL: store_factor2_64bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
-; CHECK-STREAMING-NEXT: splice z4.s, p0, { z4.s, z5.s }
-; CHECK-STREAMING-NEXT: splice z3.s, p0, { z2.s, z3.s }
-; CHECK-STREAMING-NEXT: splice z2.s, p0, { z0.s, z1.s }
-; CHECK-STREAMING-NEXT: ptrue p0.s, vl8
-; CHECK-STREAMING-NEXT: st3w { z2.s - z4.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl2
+; CHECK-STREAMING-NEXT: adrp x8, .LCPI0_0
+; CHECK-STREAMING-NEXT: splice z0.s, p0, { z0.s, z1.s }
+; CHECK-STREAMING-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-STREAMING-NEXT: tbl z0.s, { z0.s }, z1.s
+; CHECK-STREAMING-NEXT: str q0, [x0]
; CHECK-STREAMING-NEXT: ret
- %interleaved = call <24 x i32> @llvm.vector.interleave3.v24i32(
- <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
- store <24 x i32> %interleaved, ptr %ptr, align 4
+ %interleaved = call <4 x i32> @llvm.vector.interleave2.v4i32(
+ <2 x i32> %v0, <2 x i32> %v1)
+ store <4 x i32> %interleaved, ptr %ptr, align 4
ret void
}
-define void @store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
-; CHECK-COMPAT-LABEL: store_factor4:
+define void @store_factor2_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-COMPAT-LABEL: store_factor2_128bit:
; CHECK-COMPAT: // %bb.0:
-; CHECK-COMPAT-NEXT: mov z19.d, z6.d
-; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
-; CHECK-COMPAT-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-COMPAT-NEXT: mov z18.d, z4.d
-; CHECK-COMPAT-NEXT: mov z17.d, z2.d
-; CHECK-COMPAT-NEXT: mov z16.d, z0.d
-; CHECK-COMPAT-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-COMPAT-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-COMPAT-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-COMPAT-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-COMPAT-NEXT: ptrue p0.d, vl4
-; CHECK-COMPAT-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-COMPAT-NEXT: mov z2.s, z1.s[3]
+; CHECK-COMPAT-NEXT: mov z3.s, z0.s[3]
+; CHECK-COMPAT-NEXT: mov z4.s, z1.s[2]
+; CHECK-COMPAT-NEXT: mov z5.s, z0.s[2]
+; CHECK-COMPAT-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-COMPAT-NEXT: zip1 z2.s, z3.s, z2.s
+; CHECK-COMPAT-NEXT: zip1 z3.s, z5.s, z4.s
+; CHECK-COMPAT-NEXT: zip1 z1.d, z3.d, z2.d
+; CHECK-COMPAT-NEXT: stp q0, q1, [x0]
; CHECK-COMPAT-NEXT: ret
;
-; CHECK-STREAMING-LABEL: store_factor4:
+; CHECK-STREAMING-LABEL: store_factor2_128bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
-; CHECK-STREAMING-NEXT: splice z6.d, p0, { z6.d, z7.d }
-; CHECK-STREAMING-NEXT: splice z5.d, p0, { z4.d, z5.d }
-; CHECK-STREAMING-NEXT: splice z4.d, p0, { z2.d, z3.d }
-; CHECK-STREAMING-NEXT: splice z3.d, p0, { z0.d, z1.d }
-; CHECK-STREAMING-NEXT: ptrue p0.d, vl4
-; CHECK-STREAMING-NEXT: st4d { z3.d - z6.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: mov z2.s, z1.s[3]
+; CHECK-STREAMING-NEXT: mov z3.s, z0.s[3]
+; CHECK-STREAMING-NEXT: mov z4.s, z1.s[2]
+; CHECK-STREAMING-NEXT: mov z5.s, z0.s[2]
+; CHECK-STREAMING-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-STREAMING-NEXT: zip1 z2.s, z3.s, z2.s
+; CHECK-STREAMING-NEXT: zip1 z3.s, z5.s, z4.s
+; CHECK-STREAMING-NEXT: zip1 z1.d, z3.d, z2.d
+; CHECK-STREAMING-NEXT: stp q0, q1, [x0]
; CHECK-STREAMING-NEXT: ret
- %interleaved = call <16 x i64> @llvm.vector.interleave4.v16i64(
- <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
- store <16 x i64> %interleaved, ptr %ptr, align 8
+ %interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(
+ <4 x i32> %v0, <4 x i32> %v1)
+ store <8 x i32> %interleaved, ptr %ptr, align 4
ret void
}
-
-define void @store_factor2_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) #0 {
-; CHECK-COMPAT-LABEL: store_factor2_128bit:
+define void @store_factor3_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2) {
+; CHECK-COMPAT-LABEL: store_factor3_64bit:
; CHECK-COMPAT: // %bb.0:
-; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
-; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-COMPAT-NEXT: adrp x8, .LCPI2_0
-; CHECK-COMPAT-NEXT: add x8, x8, :lo12:.LCPI2_0
-; CHECK-COMPAT-NEXT: ptrue p1.s, vl8
-; CHECK-COMPAT-NEXT: splice z0.s, p0, z0.s, z1.s
-; CHECK-COMPAT-NEXT: ld1w { z1.s }, p1/z, [x8]
-; CHECK-COMPAT-NEXT: tbl z0.s, { z0.s }, z1.s
-; CHECK-COMPAT-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl2
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: st3w { z0.s - z2.s }, p0, [x0]
; CHECK-COMPAT-NEXT: ret
;
-; CHECK-STREAMING-LABEL: store_factor2_128bit:
+; CHECK-STREAMING-LABEL: store_factor3_64bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
-; CHECK-STREAMING-NEXT: adrp x8, .LCPI2_0
-; CHECK-STREAMING-NEXT: add x8, x8, :lo12:.LCPI2_0
-; CHECK-STREAMING-NEXT: ptrue p1.s, vl8
-; CHECK-STREAMING-NEXT: splice z0.s, p0, { z0.s, z1.s }
-; CHECK-STREAMING-NEXT: ld1w { z1.s }, p1/z, [x8]
-; CHECK-STREAMING-NEXT: tbl z0.s, { z0.s }, z1.s
-; CHECK-STREAMING-NEXT: st1w { z0.s }, p1, [x0]
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl2
+; CHECK-STREAMING-NEXT: st3w { z0.s - z2.s }, p0, [x0]
; CHECK-STREAMING-NEXT: ret
- %interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(
- <4 x i32> %v0, <4 x i32> %v1)
- store <8 x i32> %interleaved, ptr %ptr, align 4
+ %interleaved = call <6 x i32> @llvm.vector.interleave3.v6i32(
+ <2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2)
+ store <6 x i32> %interleaved, ptr %ptr, align 4
ret void
}
-define void @store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) #0 {
+define void @store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
; CHECK-COMPAT-LABEL: store_factor3_128bit:
; CHECK-COMPAT: // %bb.0:
; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
@@ -126,7 +109,29 @@ define void @store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i
ret void
}
-define void @store_factor4_128bit(ptr %ptr, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3) #0 {
+define void @store_factor4_64bit(ptr %ptr, <1 x i64> %v0, <1 x i64> %v1, <1 x i64> %v2, <1 x i64> %v3) {
+; CHECK-COMPAT-LABEL: store_factor4_64bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $d3 killed $d3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl1
+; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: store_factor4_64bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl1
+; CHECK-STREAMING-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <4 x i64> @llvm.vector.interleave4.v4i64(
+ <1 x i64> %v0, <1 x i64> %v1, <1 x i64> %v2, <1 x i64> %v3)
+ store <4 x i64> %interleaved, ptr %ptr, align 8
+ ret void
+}
+
+define void @store_factor4_128bit(ptr %ptr, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3) {
; CHECK-COMPAT-LABEL: store_factor4_128bit:
; CHECK-COMPAT: // %bb.0:
; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
@@ -148,104 +153,220 @@ define void @store_factor4_128bit(ptr %ptr, <2 x i64> %v0, <2 x i64> %v1, <2 x i
ret void
}
-define void @masked_store_factor3(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) #0 {
-; CHECK-COMPAT-LABEL: masked_store_factor3:
+define void @masked_store_factor2_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1, <2 x i1> %mask) {
+; CHECK-COMPAT-LABEL: masked_store_factor2_64bit:
; CHECK-COMPAT: // %bb.0:
-; CHECK-COMPAT-NEXT: mov z18.d, z4.d
-; CHECK-COMPAT-NEXT: // kill: def $d6 killed $d6 def $z6
+; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 def $z2
+; CHECK-COMPAT-NEXT: mov z3.s, z2.s[1]
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl2
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-COMPAT-NEXT: adrp x8, .LCPI6_0
+; CHECK-COMPAT-NEXT: splice z0.s, p0, z0.s, z1.s
; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
-; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-COMPAT-NEXT: mov z17.d, z2.d
-; CHECK-COMPAT-NEXT: ptrue p1.s, vl8
-; CHECK-COMPAT-NEXT: mov z16.d, z0.d
-; CHECK-COMPAT-NEXT: uunpklo z0.h, z6.b
-; CHECK-COMPAT-NEXT: splice z18.s, p0, z18.s, z5.s
-; CHECK-COMPAT-NEXT: splice z17.s, p0, z17.s, z3.s
-; CHECK-COMPAT-NEXT: splice z16.s, p0, z16.s, z1.s
-; CHECK-COMPAT-NEXT: uunpklo z0.s, z0.h
-; CHECK-COMPAT-NEXT: lsl z0.s, z0.s, #31
-; CHECK-COMPAT-NEXT: asr z0.s, z0.s, #31
-; CHECK-COMPAT-NEXT: cmpne p0.s, p1/z, z0.s, #0
-; CHECK-COMPAT-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-COMPAT-NEXT: zip1 z2.h, z2.h, z3.h
+; CHECK-COMPAT-NEXT: zip1 z2.h, z2.h, z2.h
+; CHECK-COMPAT-NEXT: uunpklo z2.s, z2.h
+; CHECK-COMPAT-NEXT: lsl z2.s, z2.s, #31
+; CHECK-COMPAT-NEXT: asr z1.s, z2.s, #31
+; CHECK-COMPAT-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
+; CHECK-COMPAT-NEXT: tbl z0.s, { z0.s }, z2.s
+; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-COMPAT-NEXT: st1w { z0.s }, p1, [x0]
; CHECK-COMPAT-NEXT: ret
;
-; CHECK-STREAMING-LABEL: masked_store_factor3:
+; CHECK-STREAMING-LABEL: masked_store_factor2_64bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: uunpklo z6.h, z6.b
+; CHECK-STREAMING-NEXT: mov z3.s, z2.s[1]
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl2
+; CHECK-STREAMING-NEXT: adrp x8, .LCPI6_0
+; CHECK-STREAMING-NEXT: splice z0.s, p0, { z0.s, z1.s }
; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
-; CHECK-STREAMING-NEXT: ptrue p1.s, vl8
-; CHECK-STREAMING-NEXT: splice z4.s, p0, { z4.s, z5.s }
-; CHECK-STREAMING-NEXT: splice z3.s, p0, { z2.s, z3.s }
-; CHECK-STREAMING-NEXT: splice z2.s, p0, { z0.s, z1.s }
-; CHECK-STREAMING-NEXT: uunpklo z6.s, z6.h
-; CHECK-STREAMING-NEXT: lsl z6.s, z6.s, #31
-; CHECK-STREAMING-NEXT: asr z6.s, z6.s, #31
-; CHECK-STREAMING-NEXT: cmpne p0.s, p1/z, z6.s, #0
-; CHECK-STREAMING-NEXT: st3w { z2.s - z4.s }, p0, [x0]
+; CHECK-STREAMING-NEXT: zip1 z2.h, z2.h, z3.h
+; CHECK-STREAMING-NEXT: zip1 z2.h, z2.h, z2.h
+; CHECK-STREAMING-NEXT: uunpklo z2.s, z2.h
+; CHECK-STREAMING-NEXT: lsl z2.s, z2.s, #31
+; CHECK-STREAMING-NEXT: asr z1.s, z2.s, #31
+; CHECK-STREAMING-NEXT: ldr q2, [x8, :lo12:.LCPI6_0]
+; CHECK-STREAMING-NEXT: tbl z0.s, { z0.s }, z2.s
+; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-STREAMING-NEXT: st1w { z0.s }, p1, [x0]
; CHECK-STREAMING-NEXT: ret
- %interleaved = call <24 x i32> @llvm.vector.interleave3.v24i32(
- <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
- %interleaved.mask = call <24 x i1> @llvm.vector.interleave3.v24i1(
- <8 x i1> %mask, <8 x i1> %mask, <8 x i1> %mask)
- call void @llvm.masked.store.v24i32.p0(<24 x i32> %interleaved, ptr %ptr,
- i32 4, <24 x i1> %interleaved.mask)
+ %interleaved = call <4 x i32> @llvm.vector.interleave2.v4i32(
+ <2 x i32> %v0, <2 x i32> %v1)
+ %interleaved.mask = call <4 x i1> @llvm.vector.interleave2.v4i1(
+ <2 x i1> %mask, <2 x i1> %mask)
+ call void @llvm.masked.store.v4i32.p0(<4 x i32> %interleaved, ptr %ptr,
+ i32 4, <4 x i1> %interleaved.mask)
ret void
}
-define void @masked_store_factor4(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) #0 {
-; CHECK-COMPAT-LABEL: masked_store_factor4:
+define void @masked_store_factor2_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i1> %mask) {
+; CHECK-COMPAT-LABEL: masked_store_factor2_128bit:
; CHECK-COMPAT: // %bb.0:
-; CHECK-COMPAT-NEXT: mov z19.d, z6.d
-; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
-; CHECK-COMPAT-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-COMPAT-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-COMPAT-NEXT: adrp x8, .LCPI7_0
+; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 def $z2
; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-COMPAT-NEXT: mov z18.d, z4.d
-; CHECK-COMPAT-NEXT: ptrue p1.d, vl4
-; CHECK-COMPAT-NEXT: mov z17.d, z2.d
-; CHECK-COMPAT-NEXT: ldr d2, [sp]
-; CHECK-COMPAT-NEXT: mov z16.d, z0.d
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-COMPAT-NEXT: mov z4.s, z1.s[3]
+; CHECK-COMPAT-NEXT: mov z5.s, z0.s[3]
+; CHECK-COMPAT-NEXT: ldr q3, [x8, :lo12:.LCPI7_0]
+; CHECK-COMPAT-NEXT: mov z6.s, z1.s[2]
+; CHECK-COMPAT-NEXT: mov z7.s, z0.s[2]
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl4
+; CHECK-COMPAT-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-COMPAT-NEXT: mov x8, #4 // =0x4
+; CHECK-COMPAT-NEXT: tbl z3.h, { z2.h }, z3.h
+; CHECK-COMPAT-NEXT: zip1 z2.h, z2.h, z2.h
+; CHECK-COMPAT-NEXT: zip1 z4.s, z5.s, z4.s
+; CHECK-COMPAT-NEXT: zip1 z5.s, z7.s, z6.s
; CHECK-COMPAT-NEXT: uunpklo z2.s, z2.h
-; CHECK-COMPAT-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-COMPAT-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-COMPAT-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-COMPAT-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-COMPAT-NEXT: uunpklo z0.d, z2.s
-; CHECK-COMPAT-NEXT: lsl z0.d, z0.d, #63
-; CHECK-COMPAT-NEXT: asr z0.d, z0.d, #63
-; CHECK-COMPAT-NEXT: cmpne p0.d, p1/z, z0.d, #0
-; CHECK-COMPAT-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-COMPAT-NEXT: uunpklo z3.s, z3.h
+; CHECK-COMPAT-NEXT: lsl z2.s, z2.s, #31
+; CHECK-COMPAT-NEXT: lsl z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: asr z2.s, z2.s, #31
+; CHECK-COMPAT-NEXT: asr z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: cmpne p2.s, p0/z, z2.s, #0
+; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-COMPAT-NEXT: zip1 z3.d, z5.d, z4.d
+; CHECK-COMPAT-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
+; CHECK-COMPAT-NEXT: st1w { z0.s }, p2, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor2_128bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: adrp x8, .LCPI7_0
+; CHECK-STREAMING-NEXT: mov z4.s, z1.s[3]
+; CHECK-STREAMING-NEXT: mov z5.s, z0.s[3]
+; CHECK-STREAMING-NEXT: ldr q3, [x8, :lo12:.LCPI7_0]
+; CHECK-STREAMING-NEXT: mov z6.s, z1.s[2]
+; CHECK-STREAMING-NEXT: mov z7.s, z0.s[2]
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl4
+; CHECK-STREAMING-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-STREAMING-NEXT: mov x8, #4 // =0x4
+; CHECK-STREAMING-NEXT: tbl z3.h, { z2.h }, z3.h
+; CHECK-STREAMING-NEXT: zip1 z2.h, z2.h, z2.h
+; CHECK-STREAMING-NEXT: zip1 z4.s, z5.s, z4.s
+; CHECK-STREAMING-NEXT: zip1 z5.s, z7.s, z6.s
+; CHECK-STREAMING-NEXT: uunpklo z2.s, z2.h
+; CHECK-STREAMING-NEXT: uunpklo z3.s, z3.h
+; CHECK-STREAMING-NEXT: lsl z2.s, z2.s, #31
+; CHECK-STREAMING-NEXT: lsl z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: asr z2.s, z2.s, #31
+; CHECK-STREAMING-NEXT: asr z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: cmpne p2.s, p0/z, z2.s, #0
+; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-STREAMING-NEXT: zip1 z3.d, z5.d, z4.d
+; CHECK-STREAMING-NEXT: st1w { z3.s }, p1, [x0, x8, lsl #2]
+; CHECK-STREAMING-NEXT: st1w { z0.s }, p2, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <8 x i32> @llvm.vector.interleave2.v8i32(
+ <4 x i32> %v0, <4 x i32> %v1)
+ %interleaved.mask = call <8 x i1> @llvm.vector.interleave2.v8i1(
+ <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v8i32.p0(<8 x i32> %interleaved, ptr %ptr,
+ i32 4, <8 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor4_64bit(ptr %ptr, <1 x i64> %v0, <1 x i64> %v1, <1 x i64> %v2, <1 x i64> %v3, <1 x i1> %mask) {
+; CHECK-COMPAT-LABEL: masked_store_factor4_64bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $w1 killed $w1 def $x1
+; CHECK-COMPAT-NEXT: sbfx x8, x1, #0, #1
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl1
+; CHECK-COMPAT-NEXT: // kill: def $d3 killed $d3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: fmov d4, x8
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; CHECK-COMPAT-NEXT: st4d { z0.d - z3.d }, p1, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor4_64bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: sbfx x8, x1, #0, #1
+; CHECK-STREAMING-NEXT: ptrue p0.d, vl1
+; CHECK-STREAMING-NEXT: fmov d4, x8
+; CHECK-STREAMING-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; CHECK-STREAMING-NEXT: st4d { z0.d - z3.d }, p1, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <4 x i64> @llvm.vector.interleave4.v4i64(
+ <1 x i64> %v0, <1 x i64> %v1, <1 x i64> %v2, <1 x i64> %v3)
+ %interleaved.mask = call <4 x i1> @llvm.vector.interleave4.v4i1(
+ <1 x i1> %mask, <1 x i1> %mask, <1 x i1> %mask, <1 x i1> %mask)
+ call void @llvm.masked.store.v4i64.p0(<4 x i64> %interleaved, ptr %ptr,
+ i32 8, <4 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor4_128bit(ptr %ptr, <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3, <2 x i1> %mask) {
+; CHECK-COMPAT-LABEL: masked_store_factor4_128bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $d4 killed $d4 def $z4
+; CHECK-COMPAT-NEXT: ptrue p0.d, vl2
+; CHECK-COMPAT-NEXT: // kill: def $q3 killed $q3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: uunpklo z4.d, z4.s
+; CHECK-COMPAT-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-COMPAT-NEXT: lsl z4.d, z4.d, #63
+; CHECK-COMPAT-NEXT: asr z4.d, z4.d, #63
+; CHECK-COMPAT-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; CHECK-COMPAT-NEXT: st4d { z0.d - z3.d }, p1, [x0]
; CHECK-COMPAT-NEXT: ret
;
-; CHECK-STREAMING-LABEL: masked_store_factor4:
+; CHECK-STREAMING-LABEL: masked_store_factor4_128bit:
; CHECK-STREAMING: // %bb.0:
-; CHECK-STREAMING-NEXT: ldr d16, [sp]
+; CHECK-STREAMING-NEXT: uunpklo z4.d, z4.s
; CHECK-STREAMING-NEXT: ptrue p0.d, vl2
-; CHECK-STREAMING-NEXT: ptrue p1.d, vl4
-; CHECK-STREAMING-NEXT: uunpklo z16.s, z16.h
-; CHECK-STREAMING-NEXT: splice z6.d, p0, { z6.d, z7.d }
-; CHECK-STREAMING-NEXT: splice z5.d, p0, { z4.d, z5.d }
-; CHECK-STREAMING-NEXT: splice z4.d, p0, { z2.d, z3.d }
-; CHECK-STREAMING-NEXT: splice z3.d, p0, { z0.d, z1.d }
-; CHECK-STREAMING-NEXT: uunpklo z16.d, z16.s
-; CHECK-STREAMING-NEXT: lsl z16.d, z16.d, #63
-; CHECK-STREAMING-NEXT: asr z7.d, z16.d, #63
-; CHECK-STREAMING-NEXT: cmpne p0.d, p1/z, z7.d, #0
-; CHECK-STREAMING-NEXT: st4d { z3.d - z6.d }, p0, [x0]
+; CHECK-STREAMING-NEXT: lsl z4.d, z4.d, #63
+; CHECK-STREAMING-NEXT: asr z4.d, z4.d, #63
+; CHECK-STREAMING-NEXT: cmpne p1.d, p0/z, z4.d, #0
+; CHECK-STREAMING-NEXT: st4d { z0.d - z3.d }, p1, [x0]
+; CHECK-STREAMING-NEXT: ret
+ %interleaved = call <8 x i64> @llvm.vector.interleave4.v8i64(
+ <2 x i64> %v0, <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %v3)
+ %interleaved.mask = call <8 x i1> @llvm.vector.interleave4.v8i1(
+ <2 x i1> %mask, <2 x i1> %mask, <2 x i1> %mask, <2 x i1> %mask)
+ call void @llvm.masked.store.v8i64.p0(<8 x i64> %interleaved, ptr %ptr,
+ i32 8, <8 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor3_64bit(ptr %ptr, <2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2, <2 x i1> %mask) {
+; CHECK-COMPAT-LABEL: masked_store_factor3_64bit:
+; CHECK-COMPAT: // %bb.0:
+; CHECK-COMPAT-NEXT: // kill: def $d3 killed $d3 def $z3
+; CHECK-COMPAT-NEXT: ptrue p0.s, vl2
+; CHECK-COMPAT-NEXT: // kill: def $d2 killed $d2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: lsl z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: // kill: def $d1 killed $d1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: // kill: def $d0 killed $d0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-COMPAT-NEXT: asr z3.s, z3.s, #31
+; CHECK-COMPAT-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-COMPAT-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-COMPAT-NEXT: ret
+;
+; CHECK-STREAMING-LABEL: masked_store_factor3_64bit:
+; CHECK-STREAMING: // %bb.0:
+; CHECK-STREAMING-NEXT: lsl z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: ptrue p0.s, vl2
+; CHECK-STREAMING-NEXT: asr z3.s, z3.s, #31
+; CHECK-STREAMING-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-STREAMING-NEXT: st3w { z0.s - z2.s }, p1, [x0]
; CHECK-STREAMING-NEXT: ret
- %interleaved = call <16 x i64> @llvm.vector.interleave4.v16i64(
- <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
- %interleaved.mask = call <16 x i1> @llvm.vector.interleave4.v16i1(
- <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
- call void @llvm.masked.store.v16i64.p0(<16 x i64> %interleaved, ptr %ptr,
- i32 8, <16 x i1> %interleaved.mask)
+ %interleaved = call <6 x i32> @llvm.vector.interleave3.v6i32(
+ <2 x i32> %v0, <2 x i32> %v1, <2 x i32> %v2)
+ %interleaved.mask = call <6 x i1> @llvm.vector.interleave3.v6i1(
+ <2 x i1> %mask, <2 x i1> %mask, <2 x i1> %mask)
+ call void @llvm.masked.store.v6i32.p0(<6 x i32> %interleaved, ptr %ptr,
+ i32 4, <6 x i1> %interleaved.mask)
ret void
}
-define void @masked_store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) #0 {
+define void @masked_store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) {
; CHECK-COMPAT-LABEL: masked_store_factor3_128bit:
; CHECK-COMPAT: // %bb.0:
; CHECK-COMPAT-NEXT: // kill: def $d3 killed $d3 def $z3
@@ -278,7 +399,6 @@ define void @masked_store_factor3_128bit(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1,
ret void
}
-attributes #0 = { vscale_range(2,16) }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK-COMPAT-IADISABLED: {{.*}}
; CHECK-COMPAT-IAENABLED: {{.*}}
More information about the llvm-commits
mailing list