[llvm] [AArch64][Vectorizer] Enable scalable factor-3 interleaving (PR #217370)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 08:24:14 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/217370
>From 923ffd8b94ca150cf38c76fc1aadb30ae7987457 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Wed, 19 Aug 2026 15:13:15 +0000
Subject: [PATCH] [AArch64][Vectorizer] Enable scalable factor-3 interleaving
AArch64 can lower scalable `llvm.vector.interleave3` and
`llvm.vector.deinterleave3` intrinsics. Allow the vectorizer to cost these
operations when the interleave factor is supported.
Factor-3 lowering requires each input vector to have a known minimum size of
384 bits, so retain an invalid cost for smaller scalable vector types.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 16 +-
.../AArch64/force-target-instruction-cost.ll | 56 ++++--
.../LoopVectorize/AArch64/interleaved_cost.ll | 177 ++++++++++++++++++
.../AArch64/sve-interleave-low-vf-cost.ll | 85 ++++++++-
.../AArch64/sve-interleaved-accesses.ll | 41 ++--
5 files changed, 324 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8f0bbc0408d97..d7e700212b6e1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5737,12 +5737,16 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (VecTy->isScalableTy() && !ST->hasSVE())
return InstructionCost::getInvalid();
- // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
- // only have lowering for power-of-2 factors.
- // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
- // InterleavedAccessPass for ld3/st3
- if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
- return InstructionCost::getInvalid();
+ // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
+ // supports factors up to the maximum supported interleave factor.
+ if (VecTy->isScalableTy()) {
+ if (Factor > TLI->getMaxSupportedInterleaveFactor())
+ return InstructionCost::getInvalid();
+
+ if (Factor == 3 &&
+ DL.getTypeSizeInBits(VecTy).getKnownMinValue() != (3 * 128))
+ return InstructionCost::getInvalid();
+ }
// Vectorization for masked interleaved accesses is only enabled for scalable
// VF.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 4bc2adc5af8fc..37c86b6dc0607 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -439,27 +439,39 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[ITER_CHECK:.*:]]
; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; COST1-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 5
+; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP25]]
+; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
+; COST1-NEXT: [[TMP26:%.*]] = shl nuw i64 [[TMP24]], 4
+; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP25]]
+; COST1-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT: [[TMP28:%.*]] = add i64 [[TMP26]], 0
+; COST1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP28]], 1
+; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[TMP5]]
; COST1-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 3
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP25]]
+; COST1-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST1: [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -509,23 +521,31 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[ITER_CHECK:.*:]]
; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; COST10-NEXT: [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP22]]
+; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
+; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP22]]
+; COST10-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
-; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST10-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
+; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP22]]
+; COST10-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST10-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST10: [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST10: [[VEC_EPILOG_PH]]:
-; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST10-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
index 7ce07a82e6465..b6639ee3d813f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
@@ -130,6 +130,183 @@ for.end:
ret void
}
+%i8.3 = type {i8, i8, i8}
+define void @i8_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i8_factor_3'
+; VF_4-LABEL: 'i8_factor_3'
+; VF_8-LABEL: 'i8_factor_3'
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i8_factor_3'
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i8, ptr %tmp0, align 1
+ %tmp4 = load i8, ptr %tmp1, align 1
+ %tmp5 = load i8, ptr %tmp2, align 1
+ store i8 %tmp3, ptr %tmp0, align 1
+ store i8 %tmp4, ptr %tmp1, align 1
+ store i8 %tmp5, ptr %tmp2, align 1
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+%i16.3 = type {i16, i16, i16}
+define void @i16_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i16_factor_3'
+; VF_4-LABEL: 'i16_factor_3'
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_8-LABEL: 'i16_factor_3'
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i16_factor_3'
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i16, ptr %tmp0, align 2
+ %tmp4 = load i16, ptr %tmp1, align 2
+ %tmp5 = load i16, ptr %tmp2, align 2
+ store i16 %tmp3, ptr %tmp0, align 2
+ store i16 %tmp4, ptr %tmp1, align 2
+ store i16 %tmp5, ptr %tmp2, align 2
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+%i32.3 = type {i32, i32, i32}
+define void @i32_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i32_factor_3'
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_4-LABEL: 'i32_factor_3'
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_8-LABEL: 'i32_factor_3'
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i32_factor_3'
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i32, ptr %tmp0, align 4
+ %tmp4 = load i32, ptr %tmp1, align 4
+ %tmp5 = load i32, ptr %tmp2, align 4
+ store i32 %tmp3, ptr %tmp0, align 4
+ store i32 %tmp4, ptr %tmp1, align 4
+ store i32 %tmp5, ptr %tmp2, align 4
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+%i64.3 = type {i64, i64, i64}
+define void @i64_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i64_factor_3'
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_4-LABEL: 'i64_factor_3'
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_8-LABEL: 'i64_factor_3'
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i64_factor_3'
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i64, ptr %tmp0, align 8
+ %tmp4 = load i64, ptr %tmp1, align 8
+ %tmp5 = load i64, ptr %tmp2, align 8
+ store i64 %tmp3, ptr %tmp0, align 8
+ store i64 %tmp4, ptr %tmp1, align 8
+ store i64 %tmp5, ptr %tmp2, align 8
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
%i64.8 = type {i64, i64, i64, i64, i64, i64, i64, i64}
define void @i64_factor_8(ptr %data, i64 %n) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
index 2bd9a42404a5b..0ef07336d1113 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
@@ -70,6 +70,48 @@ exit:
ret void
}
+define void @deinterleave3_nxv3i16_load(ptr noalias readonly %src, ptr noalias %out, i64 %n) #0 {
+; CHECK-LABEL: LV: Checking a loop in 'deinterleave3_nxv3i16_load'
+; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%ptr.b>
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sum.b = phi double[ 0.0, %entry ], [ %add.b, %loop ]
+ %sum.g = phi double[ 0.0, %entry ], [ %add.g, %loop ]
+ %sum.r = phi double[ 0.0, %entry ], [ %add.r, %loop ]
+
+ %ptr.b = getelementptr inbounds { i16, i16, i16 }, ptr %src, i64 %iv, i32 0
+ %load.b = load i16, ptr %ptr.b, align 2
+
+ %ptr.g = getelementptr inbounds { i16, i16, i16 }, ptr %src, i64 %iv, i32 1
+ %load.g = load i16, ptr %ptr.g, align 2
+
+ %ptr.r = getelementptr inbounds { i16, i16, i16 }, ptr %src, i64 %iv, i32 2
+ %load.r = load i16, ptr %ptr.r, align 2
+
+ %ext.b = uitofp i16 %load.b to double
+ %ext.g = uitofp i16 %load.g to double
+ %ext.r = uitofp i16 %load.r to double
+
+ %add.b = fadd double %sum.b, %ext.b
+ %add.g = fadd double %sum.g, %ext.g
+ %add.r = fadd double %sum.r, %ext.r
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ store double %add.b, ptr %out, align 8
+ %out1 = getelementptr inbounds double, ptr %out, i64 1
+ store double %add.g, ptr %out1, align 8
+ %out2 = getelementptr inbounds double, ptr %out, i64 2
+ store double %add.r, ptr %out2, align 8
+ ret void
+}
+
; Check that the increased low-VF interleaved-store cost prevents selection of
; an SVE epilogue.
@@ -85,7 +127,7 @@ exit:
; CHECK: LV: Selecting VF: vscale x 16
; CHECK: LEV: Vectorizing epilogue loop with VF = 8
define void @deinterleave4_nxv4i16_load_interleave4_nxv4i8_store(
- ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
+ ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
entry:
%empty = icmp eq i32 %n, 0
br i1 %empty, label %exit, label %loop
@@ -130,4 +172,45 @@ exit:
ret void
}
+; CHECK-LABEL: LV: Checking a loop in 'deinterleave3_nxv3i16_load_interleave3_nxv3i16_store'
+; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%ptr.b>
+; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%out.b>
+define void @deinterleave3_nxv3i16_load_interleave3_nxv3i16_store(
+ ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
+entry:
+ %empty = icmp eq i32 %n, 0
+ br i1 %empty, label %exit, label %loop
+
+loop:
+ %src.iv = phi ptr [ %src.next, %loop ], [ %src, %entry ]
+ %out.iv = phi ptr [ %out.next, %loop ], [ %out, %entry ]
+ %iv = phi i32 [ %iv.next, %loop ], [ %n, %entry ]
+
+ %ptr.b = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 0, i32 0
+ %ptr.g = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 0, i32 1
+ %ptr.r = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 0, i32 2
+ %load.b = load i16, ptr %ptr.b, align 2
+ %load.g = load i16, ptr %ptr.g, align 2
+ %load.r = load i16, ptr %ptr.r, align 2
+
+ %shift.b = lshr i16 %load.b, 8
+ %shift.g = lshr i16 %load.g, 8
+ %shift.r = lshr i16 %load.r, 8
+ %out.b = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 0, i32 0
+ %out.g = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 0, i32 1
+ %out.r = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 0, i32 2
+ store i16 %shift.b, ptr %out.b, align 2
+ store i16 %shift.g, ptr %out.g, align 2
+ store i16 %shift.r, ptr %out.r, align 2
+
+ %src.next = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 1
+ %out.next = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 1
+ %iv.next = add nsw i32 %iv, -1
+ %done = icmp eq i32 %iv.next, 0
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 26ca4b60a00de..a2681ddab7db6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1311,9 +1311,6 @@ end:
; dst[i].y = a[i].y - b[i].y;
; dst[i].z = a[i].z << b[i].z;
; }
-;
-; TODO: Support scalable interleave groups once we can also codegen
-; @llvm.[de]interleave3
%struct.xyz = type { i32, i32, i32 }
define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {
@@ -1326,36 +1323,28 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP1]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 12 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x ptr> align 4 [[WIDE_GEP3]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP4:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER5]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT: [[WIDE_GEP8:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 4
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP5]], <vscale x 4 x ptr> align 4 [[WIDE_GEP8]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP9:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i32> [[WIDE_MASKED_GATHER10]], [[WIDE_MASKED_GATHER12]]
-; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 8
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x ptr> align 4 [[WIDE_GEP13]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: middle.block:
More information about the llvm-commits
mailing list