[llvm] [AArch64] Support wide interleaved store combine (PR #217856)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 02:15:57 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Kamlesh Kumar (kamleshbhalui)
<details>
<summary>Changes</summary>
Splitting fixed-length interleaves whose inputs are wider than a NEON register into a optimal sequence of stN to prevent type legalization from splitting the interleave itself, leaving ZIPs and plain stores that can no longer be recognized as an interleaved store.
---
Patch is 36.09 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217856.diff
5 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+59-14)
- (modified) llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll (+11-20)
- (modified) llvm/test/CodeGen/AArch64/interleaved-accesses.ll (+24-71)
- (modified) llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll (+33-60)
- (modified) llvm/test/CodeGen/AArch64/vector-interleave-store.ll (+345)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 36a86241fe3b2..bdfdd4b0c410f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -28013,8 +28013,6 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
EVT SubVecTy = ValueInterleaveOps[0].getValueType();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- if (!TLI.isTypeLegal(SubVecTy))
- return SDValue();
bool IsScalable = SubVecTy.isScalableVector();
unsigned SubBits = SubVecTy.getSizeInBits().getKnownMinValue();
if (IsScalable && SubBits != 128)
@@ -28022,6 +28020,8 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
auto *MemN = cast<MemSDNode>(N);
if (IsScalable) {
+ if (!TLI.isTypeLegal(SubVecTy))
+ return SDValue();
SDValue Pred;
if (IsMasked) {
Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
@@ -28055,26 +28055,71 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
// Fixed length vector using NEON
- if (!IsMasked && (SubBits == 64 || SubBits == 128) &&
- Subtarget.isNeonAvailable()) {
+ if (!IsMasked && Subtarget.isNeonAvailable()) {
static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2,
Intrinsic::aarch64_neon_st3,
Intrinsic::aarch64_neon_st4};
- SmallVector<SDValue, 8> Ops;
- Ops.push_back(Chain);
- Ops.push_back(
- DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64));
- Ops.append(ValueInterleaveOps);
- Ops.push_back(BasePtr);
- return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
- DAG.getVTList(MVT::Other), Ops,
- MemN->getMemoryVT(), MemN->getMemOperand());
+ SDValue IID = DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64);
+ // Split fixed-length interleaves whose inputs are wider than a NEON
+ // register into a sequence of stN. Type legalization would
+ // otherwise split the interleave itself, leaving ZIPs and plain stores
+ // that can no longer be recognized as an interleaved store.
+
+ if (SubBits % 64 != 0)
+ return SDValue();
+
+ EVT EltVT = SubVecTy.getVectorElementType();
+ unsigned EltBits = EltVT.getSizeInBits();
+ auto IsLegalChunk = [&](unsigned ChunkBits) {
+ if (ChunkBits % EltBits != 0)
+ return false;
+ EVT ChunkVT =
+ EVT::getVectorVT(*DAG.getContext(), EltVT, ChunkBits / EltBits);
+ return TLI.isTypeLegal(ChunkVT);
+ };
+
+ unsigned Num128BitChunks = SubBits / 128;
+ bool Has64BitTail = SubBits % 128 != 0;
+ if ((Num128BitChunks && !IsLegalChunk(128)) ||
+ (Has64BitTail && !IsLegalChunk(64)))
+ return SDValue();
+
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineMemOperand *MMO = MemN->getMemOperand();
+ SmallVector<SDValue, 8> StoreChains;
+ unsigned NumChunks = Num128BitChunks + Has64BitTail;
+ StoreChains.reserve(NumChunks);
+ unsigned Offset = 0;
+ unsigned EltOffset = 0;
+ for (unsigned I = 0; I != NumChunks; ++I) {
+ unsigned ChunkBits = I < Num128BitChunks ? 128 : 64;
+ unsigned NarrowNumElts = ChunkBits / EltBits;
+ EVT NarrowVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NarrowNumElts);
+ unsigned ChunkBytes = NumParts * ChunkBits / 8;
+ EVT ChunkMemVT =
+ EVT::getVectorVT(*DAG.getContext(), EltVT, NarrowNumElts * NumParts);
+ SDValue Ptr =
+ DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(Offset));
+ SmallVector<SDValue, 8> Ops = {Chain, IID};
+ for (SDValue V : ValueInterleaveOps)
+ Ops.push_back(DAG.getExtractSubvector(DL, NarrowVT, V, EltOffset));
+ Ops.push_back(Ptr);
+ MachineMemOperand *ChunkMMO =
+ MF.getMachineMemOperand(MMO, Offset, ChunkBytes);
+ StoreChains.push_back(DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
+ DAG.getVTList(MVT::Other),
+ Ops, ChunkMemVT, ChunkMMO));
+ Offset += ChunkBytes;
+ EltOffset += NarrowNumElts;
+ }
+ return DAG.getTokenFactor(DL, StoreChains);
}
// Fixed-length vectors using SVE
if (!Subtarget.isSVEorStreamingSVEAvailable())
return SDValue();
-
+ if (!TLI.isTypeLegal(SubVecTy))
+ return SDValue();
EVT ContainerVT = getContainerForFixedLengthVector(DAG, SubVecTy);
SDValue Pred;
if (IsMasked) {
diff --git a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
index 1278606f5b63f..4bdd5bfb4588d 100644
--- a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
@@ -29,17 +29,13 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
; CHECK-LABEL: store_factor3_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v4.16b, v3.16b
; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
-; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
-; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: mov v3.16b, v1.16b
+; CHECK-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; CHECK-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
store <24 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -49,21 +45,16 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) vscale_range(2,2) {
; CHECK-LABEL: store_factor4_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
; CHECK-NEXT: mov v19.16b, v6.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v6.16b, v5.16b
; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v5.16b, v3.16b
; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: mov v4.16b, v1.16b
+; CHECK-NEXT: st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT: st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
store <16 x i64> %interleaved.vec, ptr %ptr, align 4
diff --git a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
index bedce9ec6a86e..2e52b3b373484 100644
--- a/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/interleaved-accesses.ll
@@ -2331,40 +2331,17 @@ define void @store_factor3_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
}
define void @store_factor3_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
-; NEON-IAENABLED-LABEL: store_factor3_wide_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
-; NEON-IAENABLED-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
-; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
-; NEON-IAENABLED-NEXT: mov v4.16b, v3.16b
-; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
-; NEON-IAENABLED-NEXT: mov v3.16b, v1.16b
-; NEON-IAENABLED-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
-; NEON-IAENABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: store_factor3_wide_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: sub sp, sp, #96
-; NEON-IADISABLED-NEXT: .cfi_def_cfa_offset 96
-; NEON-IADISABLED-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
-; NEON-IADISABLED-NEXT: mov v18.16b, v4.16b
-; NEON-IADISABLED-NEXT: mov x8, sp
-; NEON-IADISABLED-NEXT: mov v17.16b, v2.16b
-; NEON-IADISABLED-NEXT: mov v4.16b, v3.16b
-; NEON-IADISABLED-NEXT: mov v16.16b, v0.16b
-; NEON-IADISABLED-NEXT: mov v3.16b, v1.16b
-; NEON-IADISABLED-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x8]
-; NEON-IADISABLED-NEXT: add x8, sp, #48
-; NEON-IADISABLED-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x8]
-; NEON-IADISABLED-NEXT: ldp q2, q0, [sp, #64]
-; NEON-IADISABLED-NEXT: ldp q5, q1, [sp, #32]
-; NEON-IADISABLED-NEXT: ldp q3, q4, [sp]
-; NEON-IADISABLED-NEXT: stp q2, q0, [x0, #64]
-; NEON-IADISABLED-NEXT: stp q5, q1, [x0, #32]
-; NEON-IADISABLED-NEXT: stp q3, q4, [x0]
-; NEON-IADISABLED-NEXT: add sp, sp, #96
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: store_factor3_wide_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: mov v18.16b, v4.16b
+; NEON-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
+; NEON-NEXT: mov v17.16b, v2.16b
+; NEON-NEXT: mov v4.16b, v3.16b
+; NEON-NEXT: mov v16.16b, v0.16b
+; NEON-NEXT: mov v3.16b, v1.16b
+; NEON-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; NEON-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
+; NEON-NEXT: ret
;
; NO_NEON-LABEL: store_factor3_wide_intrinsic:
; NO_NEON: // %bb.0:
@@ -2458,43 +2435,19 @@ define void @store_factor4_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4
}
define void @store_factor4_wide_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
-; NEON-IAENABLED-LABEL: store_factor4_wide_intrinsic:
-; NEON-IAENABLED: // %bb.0:
-; NEON-IAENABLED-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
-; NEON-IAENABLED-NEXT: mov v19.16b, v6.16b
-; NEON-IAENABLED-NEXT: mov v18.16b, v4.16b
-; NEON-IAENABLED-NEXT: mov v6.16b, v5.16b
-; NEON-IAENABLED-NEXT: mov v17.16b, v2.16b
-; NEON-IAENABLED-NEXT: mov v5.16b, v3.16b
-; NEON-IAENABLED-NEXT: mov v16.16b, v0.16b
-; NEON-IAENABLED-NEXT: mov v4.16b, v1.16b
-; NEON-IAENABLED-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
-; NEON-IAENABLED-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
-; NEON-IAENABLED-NEXT: ret
-;
-; NEON-IADISABLED-LABEL: store_factor4_wide_intrinsic:
-; NEON-IADISABLED: // %bb.0:
-; NEON-IADISABLED-NEXT: zip2 v16.4s, v3.4s, v7.4s
-; NEON-IADISABLED-NEXT: zip2 v17.4s, v1.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip1 v3.4s, v3.4s, v7.4s
-; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip2 v5.4s, v2.4s, v6.4s
-; NEON-IADISABLED-NEXT: zip2 v7.4s, v0.4s, v4.4s
-; NEON-IADISABLED-NEXT: zip1 v2.4s, v2.4s, v6.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v4.4s
-; NEON-IADISABLED-NEXT: zip2 v18.4s, v17.4s, v16.4s
-; NEON-IADISABLED-NEXT: zip1 v16.4s, v17.4s, v16.4s
-; NEON-IADISABLED-NEXT: zip2 v4.4s, v1.4s, v3.4s
-; NEON-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
-; NEON-IADISABLED-NEXT: zip2 v3.4s, v7.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip1 v5.4s, v7.4s, v5.4s
-; NEON-IADISABLED-NEXT: zip2 v6.4s, v0.4s, v2.4s
-; NEON-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v2.4s
-; NEON-IADISABLED-NEXT: stp q16, q18, [x0, #96]
-; NEON-IADISABLED-NEXT: stp q1, q4, [x0, #64]
-; NEON-IADISABLED-NEXT: stp q0, q6, [x0]
-; NEON-IADISABLED-NEXT: stp q5, q3, [x0, #32]
-; NEON-IADISABLED-NEXT: ret
+; NEON-LABEL: store_factor4_wide_intrinsic:
+; NEON: // %bb.0:
+; NEON-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
+; NEON-NEXT: mov v19.16b, v6.16b
+; NEON-NEXT: mov v18.16b, v4.16b
+; NEON-NEXT: mov v6.16b, v5.16b
+; NEON-NEXT: mov v17.16b, v2.16b
+; NEON-NEXT: mov v5.16b, v3.16b
+; NEON-NEXT: mov v16.16b, v0.16b
+; NEON-NEXT: mov v4.16b, v1.16b
+; NEON-NEXT: st4 { v16.4s, v17.4s, v18.4s, v19.4s }, [x0], #64
+; NEON-NEXT: st4 { v4.4s, v5.4s, v6.4s, v7.4s }, [x0]
+; NEON-NEXT: ret
;
; NO_NEON-LABEL: store_factor4_wide_intrinsic:
; NO_NEON: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
index 30f944e0a050d..faa49f7bb25b9 100644
--- a/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/CodeGen/AArch64/sve-interleaved-accesses.ll
@@ -1678,17 +1678,13 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
; CHECK-LABEL: store_factor3_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v4.16b, v3.16b
; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
-; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
-; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: mov v3.16b, v1.16b
+; CHECK-NEXT: st3 { v16.4s, v17.4s, v18.4s }, [x0], #48
+; CHECK-NEXT: st3 { v3.4s, v4.4s, v5.4s }, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
store <24 x i32> %interleaved.vec, ptr %ptr, align 4
@@ -1698,21 +1694,16 @@ define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8
define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
; CHECK-LABEL: store_factor4_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
; CHECK-NEXT: mov v19.16b, v6.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v6.16b, v5.16b
; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v5.16b, v3.16b
; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: mov v4.16b, v1.16b
+; CHECK-NEXT: st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT: st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
store <16 x i64> %interleaved.vec, ptr %ptr, align 4
@@ -1746,17 +1737,13 @@ define void @store_ptrvec_factor3_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %
; CHECK-LABEL: store_ptrvec_factor3_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT: mov v18.16b, v4.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q5 killed $q5 killed $q3_q4_q5 def $q3_q4_q5
; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v4.16b, v3.16b
; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st3d { z16.d - z18.d }, p0, [x0]
+; CHECK-NEXT: mov v3.16b, v1.16b
+; CHECK-NEXT: st3 { v16.2d, v17.2d, v18.2d }, [x0], #48
+; CHECK-NEXT: st3 { v3.2d, v4.2d, v5.2d }, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <12 x ptr> @llvm.vector.interleave3.v12p0(<4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2)
store <12 x ptr> %interleaved.vec, ptr %ptr, align 4
@@ -1766,21 +1753,16 @@ define void @store_ptrvec_factor3_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %
define void @store_ptrvec_factor4_intrinsic(ptr %ptr, <4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3) #0 {
; CHECK-LABEL: store_ptrvec_factor4_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
; CHECK-NEXT: mov v19.16b, v6.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v6.16b, v5.16b
; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v5.16b, v3.16b
; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: mov v4.16b, v1.16b
+; CHECK-NEXT: st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT: st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <16 x ptr> @llvm.vector.interleave4.v16p0(<4 x ptr> %v0, <4 x ptr> %v1, <4 x ptr> %v2, <4 x ptr> %v3)
store <16 x ptr> %interleaved.vec, ptr %ptr, align 4
@@ -1871,21 +1853,16 @@ define void @store_min_ge_type_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1)
define void @store_double_factor4_intrinsic(ptr %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) #0 {
; CHECK-LABEL: store_double_factor4_intrinsic:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q7 killed $q7 killed $q4_q5_q6_q7 def $q4_q5_q6_q7
; CHECK-NEXT: mov v19.16b, v6.16b
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
-; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
-; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v6.16b, v5.16b
; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v5.16b, v3.16b
; CHECK-NEXT: mov v16.16b, v0.16b
-; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
-; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
-; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
-; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: mov v4.16b, v1.16b
+; CHECK-NEXT: st4 { v16.2d, v17.2d, v18.2d, v19.2d }, [x0], #64
+; CHECK-NEXT: st4 { v4.2d, v5.2d, v6.2d, v7.2d }, [x0]
; CHECK-NEXT: ret
%interleaved.vec = call <16 x double> @llvm.vector.interleave4.v16f64(<4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3)
store <16 x double> %interleaved.vec, ptr %ptr, align 4
@@ -1896,17 +1873,13 @@ define void @store_float_factor3_intrinsic(ptr %ptr, <8 x float> %v0, <8 x float
; CHECK-LABEL: store_float_factor3_intrinsic:
; CHECK: // %bb.0:
; CHECK-NEXT...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/217856
More information about the llvm-commits
mailing list