[llvm] [AArch64][Vectorizer] Enable scalable factor-3 interleaving (PR #217370)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 04:03:08 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/217370
>From 0ba250f8d7e40adf4d9e9520a1fba956637d4cba Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Thu, 20 Aug 2026 08:39:46 +0000
Subject: [PATCH 1/2] [AArch64][NFC] Refactor SVE tail-folding option
interleave test for interleave 4
Replace the interleave3 test case with interleave4, since interleave3 is not
yet modelled by the cost model.
---
.../AArch64/sve-tail-folding-option.ll | 41 ++++++++++---------
1 file changed, 22 insertions(+), 19 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index 9f5937c440595..7e8ff9d62ad61 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -283,39 +283,39 @@ for.end:
define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
; CHECK-NOTF-LABEL: @interleave(
; CHECK-NOTF: vector.body:
-; CHECK-NOTF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-NOTF: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-NOTF: %{{.*}} = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> %{{.*}}, <vscale x 4 x float> %{{.*}}, <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> splat (float 4.000000e+00))
; CHECK-TF-LABEL: @interleave(
; CHECK-TF: vector.body:
-; CHECK-TF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF: %{{.*}} = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> %{{.*}}, <vscale x 4 x float> %{{.*}}, <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> splat (float 4.000000e+00))
; CHECK-TF-NORED-LABEL: @interleave(
; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NORED: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF-NORED: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF-NORED: %{{.*}} = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> %{{.*}}, <vscale x 4 x float> %{{.*}}, <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> splat (float 4.000000e+00))
; CHECK-TF-NOREC-LABEL: @interleave(
; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREC: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF-NOREC: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF-NOREC: %{{.*}} = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> %{{.*}}, <vscale x 4 x float> %{{.*}}, <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> splat (float 4.000000e+00))
; CHECK-TF-NOREV-LABEL: @interleave(
; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREV: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF-NOREV: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF-NOREV: %{{.*}} = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> %{{.*}}, <vscale x 4 x float> %{{.*}}, <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> splat (float 4.000000e+00))
; CHECK-NEOVERSE-V1-LABEL: @interleave(
; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-NEOVERSE-V1: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-NEOVERSE-V1: %{{.*}} = call <vscale x 16 x float> @llvm.vector.interleave4.nxv16f32(<vscale x 4 x float> %{{.*}}, <vscale x 4 x float> %{{.*}}, <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> splat (float 4.000000e+00))
entry:
br label %for.body
@@ -325,7 +325,7 @@ for.body:
%mul = shl nuw nsw i64 %i.021, 1
%arrayidx = getelementptr inbounds float, ptr %src, i64 %mul
%0 = load float, ptr %arrayidx, align 4
- %mul1 = mul nuw nsw i64 %i.021, 3
+ %mul1 = mul nuw nsw i64 %i.021, 4
%arrayidx2 = getelementptr inbounds float, ptr %dst, i64 %mul1
store float %0, ptr %arrayidx2, align 4
%add = or disjoint i64 %mul, 1
@@ -337,6 +337,9 @@ for.body:
%add9 = add nuw nsw i64 %mul1, 2
%arrayidx10 = getelementptr inbounds float, ptr %dst, i64 %add9
store float 3.000000e+00, ptr %arrayidx10, align 4
+ %add10 = add nuw nsw i64 %mul1, 3
+ %arrayidx11 = getelementptr inbounds float, ptr %dst, i64 %add10
+ store float 4.000000e+00, ptr %arrayidx11, align 4
%inc = add nuw nsw i64 %i.021, 1
%exitcond.not = icmp eq i64 %inc, %n
br i1 %exitcond.not, label %for.end, label %for.body
>From a5c6f85054a94b8ec1b7f9c94e1e0c7f09052890 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Wed, 19 Aug 2026 15:13:15 +0000
Subject: [PATCH 2/2] [AArch64][Vectorizer] Enable scalable factor-3
interleaving
AArch64 can lower scalable `llvm.vector.interleave3` and
`llvm.vector.deinterleave3` intrinsics. Allow the vectorizer to cost these
operations when the interleave factor is supported.
Factor-3 lowering requires each input vector to have a known minimum size of
384 bits, so retain an invalid cost for smaller scalable vector types.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 16 ++++--
.../AArch64/force-target-instruction-cost.ll | 56 +++++++++++++------
.../AArch64/sve-interleaved-accesses.ll | 38 +++++--------
3 files changed, 63 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a811764136dd6..a61512e800051 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5738,12 +5738,16 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (VecTy->isScalableTy() && !ST->hasSVE())
return InstructionCost::getInvalid();
- // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
- // only have lowering for power-of-2 factors.
- // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
- // InterleavedAccessPass for ld3/st3
- if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
- return InstructionCost::getInvalid();
+ // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
+ // supports factors up to the maximum supported interleave factor.
+ if (VecTy->isScalableTy()) {
+ if (Factor > TLI->getMaxSupportedInterleaveFactor())
+ return InstructionCost::getInvalid();
+
+ if (Factor == 3 &&
+ DL.getTypeSizeInBits(VecTy).getKnownMinValue() != (3 * 128))
+ return InstructionCost::getInvalid();
+ }
// Vectorization for masked interleaved accesses is only enabled for scalable
// VF.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 4bc2adc5af8fc..37c86b6dc0607 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -439,27 +439,39 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[ITER_CHECK:.*:]]
; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; COST1-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 5
+; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP25]]
+; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
+; COST1-NEXT: [[TMP26:%.*]] = shl nuw i64 [[TMP24]], 4
+; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP25]]
+; COST1-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT: [[TMP28:%.*]] = add i64 [[TMP26]], 0
+; COST1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP28]], 1
+; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[TMP5]]
; COST1-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 3
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP25]]
+; COST1-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST1: [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -509,23 +521,31 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[ITER_CHECK:.*:]]
; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; COST10-NEXT: [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP22]]
+; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
+; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP22]]
+; COST10-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
-; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST10-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
+; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP22]]
+; COST10-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST10-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST10: [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST10: [[VEC_EPILOG_PH]]:
-; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST10-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 26ca4b60a00de..206130a5b2468 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1326,36 +1326,28 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP1]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 12 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x ptr> align 4 [[WIDE_GEP3]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP4:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER5]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT: [[WIDE_GEP8:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 4
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP5]], <vscale x 4 x ptr> align 4 [[WIDE_GEP8]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP9:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i32> [[WIDE_MASKED_GATHER10]], [[WIDE_MASKED_GATHER12]]
-; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 8
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x ptr> align 4 [[WIDE_GEP13]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: middle.block:
More information about the llvm-commits
mailing list