[llvm] [AArch64][CostModel] Updated inserelement/extractelement costs (PR #220627)
Utpal Bora via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 00:57:38 PDT 2026
https://github.com/utpalbora updated https://github.com/llvm/llvm-project/pull/220627
>From 8355604c1b6827dc56717e7017382dabddc7fc42 Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Wed, 2 Sep 2026 14:46:41 +0000
Subject: [PATCH 1/8] pre-commit test
---
.../sve-non-native-interleaved-accesses.ll | 93 +++++++++++++++++++
1 file changed, 93 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
new file mode 100644
index 0000000000000..6a5491bca5644
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 \
+; RUN: -S < %s | FileCheck %s
+
+; Check vectorization on non-native interleaving factor 5
+define float @float_factor_5_deinterleave_cost(ptr %data, i64 %n) vscale_range(2, 0) {
+; CHECK-LABEL: define float @float_factor_5_deinterleave_cost(
+; CHECK-SAME: ptr [[DATA:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP0]], 15
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP3]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP2]], i64 16, i64 [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP5]], 20
+; CHECK-NEXT: [[TMP10:%.*]] = mul nuw i64 [[TMP6]], 20
+; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP10]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <40 x float>, ptr [[TMP36]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <40 x float> [[WIDE_VEC]], <40 x float> poison, <8 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35>
+; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <40 x float>, ptr [[TMP39]], align 4
+; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <40 x float> [[WIDE_VEC2]], <40 x float> poison, <8 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35>
+; CHECK-NEXT: [[TMP15:%.*]] = fmul <8 x float> zeroinitializer, [[STRIDED_VEC]]
+; CHECK-NEXT: [[TMP16:%.*]] = fmul <8 x float> zeroinitializer, [[STRIDED_VEC3]]
+; CHECK-NEXT: [[TMP11]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP15]])
+; CHECK-NEXT: [[TMP12]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI1]], <8 x float> [[TMP16]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 16
+; CHECK-NEXT: [[TMP13:%.*]] = fcmp uno <8 x float> [[TMP15]], [[TMP16]]
+; CHECK-NEXT: [[TMP14:%.*]] = freeze <8 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP14]])
+; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
+; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP29]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP11]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP29]], <8 x float> [[VEC_PHI1]], <8 x float> [[TMP12]]
+; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP29]], i64 [[TMP5]], i64 [[N_VEC]]
+; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[TMP18]], <8 x float> [[TMP19]])
+; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[RDX_MINMAX]])
+; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[MAX_REC:%.*]] = phi float [ [[MAX_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[BYTE_OFFSET:%.*]] = mul nuw i64 [[IV]], 20
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[BYTE_OFFSET]]
+; CHECK-NEXT: [[LOAD:%.*]] = load float, ptr [[PTR]], align 4
+; CHECK-NEXT: [[PRODUCT:%.*]] = fmul float 0.000000e+00, [[LOAD]]
+; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX_REC]], float [[PRODUCT]])
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[FOR_BODY]] ]
+; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+ %max.rec = phi float [ %max.next, %for.body ], [ 0.000000e+00, %entry ]
+ %byte.offset = mul nuw i64 %iv, 20
+ %ptr = getelementptr i8, ptr %data, i64 %byte.offset
+ %load = load float, ptr %ptr, align 4
+ %product = fmul float 0.000000e+00, %load
+ %max.next = call float @llvm.maxnum.f32(float %max.rec, float %product)
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv, %n
+ br i1 %done, label %for.end, label %for.body
+
+for.end: ; preds = %for.body
+ ret float %max.next
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
>From c296e7d766d5ebd3d5bbc7ba7a02594efe677514 Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Wed, 2 Sep 2026 14:53:20 +0000
Subject: [PATCH 2/8] [AArch64][LV] Prohibit non-native [de]interleaved pattern
for fixed-length SVE
---
.../AArch64/AArch64TargetTransformInfo.cpp | 11 ++-
.../sve-non-native-interleaved-accesses.ll | 74 ++++++++++++-------
2 files changed, 54 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8f0bbc0408d97..60c36e1797fa5 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5744,12 +5744,17 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
return InstructionCost::getInvalid();
+ auto LT = getTypeLegalizationCost(VecTy);
+ unsigned MaxNativeInterleaveFactor = TLI->getMaxSupportedInterleaveFactor();
// Vectorization for masked interleaved accesses is only enabled for scalable
- // VF.
- if (!VecTy->isScalableTy() && (UseMaskForCond || UseMaskForGaps))
+ // VF. For fixed-length SVE, avoid non-native interleave factor because
+ // the generic fallback costs wide fixed-vector shuffles too optimistically.
+ if (!VecTy->isScalableTy() && (UseMaskForCond || UseMaskForGaps ||
+ (Factor > MaxNativeInterleaveFactor &&
+ TLI->useSVEForFixedLengthVectorVT(LT.second))))
return InstructionCost::getInvalid();
- if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
+ if (!UseMaskForGaps && Factor <= MaxNativeInterleaveFactor) {
ElementCount EC = VecVTy->getElementCount();
auto *SubVecTy = VectorType::get(VecVTy->getElementType(),
EC.divideCoefficientBy(Factor));
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
index 6a5491bca5644..44d46efda678c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -8,48 +8,66 @@ define float @float_factor_5_deinterleave_cost(ptr %data, i64 %n) vscale_range(2
; CHECK-SAME: ptr [[DATA:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], 16
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP0]], 15
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP3]], 0
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[TMP2]], i64 16, i64 [[TMP3]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 8
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP5]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP5]], 3
; CHECK-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP5]], 20
; CHECK-NEXT: [[TMP10:%.*]] = mul nuw i64 [[TMP6]], 20
+; CHECK-NEXT: [[TMP7:%.*]] = mul nuw i64 [[TMP3]], 20
+; CHECK-NEXT: [[TMP8:%.*]] = mul nuw i64 [[TMP4]], 20
; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP9]]
; CHECK-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP10]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <40 x float>, ptr [[TMP36]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <40 x float> [[WIDE_VEC]], <40 x float> poison, <8 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35>
-; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <40 x float>, ptr [[TMP39]], align 4
-; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <40 x float> [[WIDE_VEC2]], <40 x float> poison, <8 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35>
-; CHECK-NEXT: [[TMP15:%.*]] = fmul <8 x float> zeroinitializer, [[STRIDED_VEC]]
-; CHECK-NEXT: [[TMP16:%.*]] = fmul <8 x float> zeroinitializer, [[STRIDED_VEC3]]
-; CHECK-NEXT: [[TMP11]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP15]])
-; CHECK-NEXT: [[TMP12]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI1]], <8 x float> [[TMP16]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 16
-; CHECK-NEXT: [[TMP13:%.*]] = fcmp uno <8 x float> [[TMP15]], [[TMP16]]
-; CHECK-NEXT: [[TMP14:%.*]] = freeze <8 x i1> [[TMP13]]
-; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP14]])
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[TMP36]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = load float, ptr [[TMP39]], align 4
+; CHECK-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[TMP17:%.*]] = fmul float 0.000000e+00, [[TMP13]]
+; CHECK-NEXT: [[TMP18:%.*]] = fmul float 0.000000e+00, [[TMP14]]
+; CHECK-NEXT: [[TMP19:%.*]] = fmul float 0.000000e+00, [[TMP15]]
+; CHECK-NEXT: [[TMP40:%.*]] = fmul float 0.000000e+00, [[TMP16]]
+; CHECK-NEXT: [[TMP21]] = call float @llvm.maxnum.f32(float [[VEC_PHI]], float [[TMP17]])
+; CHECK-NEXT: [[TMP22]] = call float @llvm.maxnum.f32(float [[VEC_PHI1]], float [[TMP18]])
+; CHECK-NEXT: [[TMP23]] = call float @llvm.maxnum.f32(float [[VEC_PHI2]], float [[TMP19]])
+; CHECK-NEXT: [[TMP24]] = call float @llvm.maxnum.f32(float [[VEC_PHI3]], float [[TMP40]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
+; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno float [[TMP17]], [[TMP18]]
+; CHECK-NEXT: [[TMP26:%.*]] = fcmp uno float [[TMP19]], [[TMP40]]
+; CHECK-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP25]]
+; CHECK-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP26]]
+; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP18:%.*]] = select i1 [[TMP29]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP11]]
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP29]], <8 x float> [[VEC_PHI1]], <8 x float> [[TMP12]]
+; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP29]], float [[VEC_PHI]], float [[TMP21]]
+; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP29]], float [[VEC_PHI1]], float [[TMP22]]
+; CHECK-NEXT: [[TMP34:%.*]] = select i1 [[TMP29]], float [[VEC_PHI2]], float [[TMP23]]
+; CHECK-NEXT: [[TMP35:%.*]] = select i1 [[TMP29]], float [[VEC_PHI3]], float [[TMP24]]
; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP29]], i64 [[TMP5]], i64 [[N_VEC]]
-; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[TMP18]], <8 x float> [[TMP19]])
-; CHECK-NEXT: [[TMP21:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[RDX_MINMAX]])
-; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call float @llvm.maxnum.f32(float [[TMP32]], float [[TMP33]])
+; CHECK-NEXT: [[RDX_MINMAX4:%.*]] = call float @llvm.maxnum.f32(float [[RDX_MINMAX]], float [[TMP34]])
+; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call float @llvm.maxnum.f32(float [[RDX_MINMAX4]], float [[TMP35]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: [[TMP37:%.*]] = xor i1 [[TMP29]], true
+; CHECK-NEXT: [[TMP38:%.*]] = and i1 [[CMP_N]], [[TMP37]]
+; CHECK-NEXT: br i1 [[TMP38]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -61,9 +79,9 @@ define float @float_factor_5_deinterleave_cost(ptr %data, i64 %n) vscale_range(2
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX_REC]], float [[PRODUCT]])
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[FOR_BODY]] ], [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -89,5 +107,5 @@ for.end: ; preds = %for.body
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
;.
>From 60e0c7504f3900ad77a87c19abe392f1dd835391 Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Fri, 4 Sep 2026 11:37:17 +0000
Subject: [PATCH 3/8] fixed formatting issues. renamed function in the testcase
to reflect the functionality
---
.../AArch64/AArch64TargetTransformInfo.cpp | 7 +-
.../sve-non-native-interleaved-accesses.ll | 167 +++++++++++-------
2 files changed, 111 insertions(+), 63 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 60c36e1797fa5..561067a3b0660 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5749,9 +5749,10 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
// Vectorization for masked interleaved accesses is only enabled for scalable
// VF. For fixed-length SVE, avoid non-native interleave factor because
// the generic fallback costs wide fixed-vector shuffles too optimistically.
- if (!VecTy->isScalableTy() && (UseMaskForCond || UseMaskForGaps ||
- (Factor > MaxNativeInterleaveFactor &&
- TLI->useSVEForFixedLengthVectorVT(LT.second))))
+ if (!VecTy->isScalableTy() &&
+ (UseMaskForCond || UseMaskForGaps ||
+ (Factor > MaxNativeInterleaveFactor &&
+ TLI->useSVEForFixedLengthVectorVT(LT.second))))
return InstructionCost::getInvalid();
if (!UseMaskForGaps && Factor <= MaxNativeInterleaveFactor) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
index 44d46efda678c..026e5ccfc427f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -2,72 +2,119 @@
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 \
; RUN: -S < %s | FileCheck %s
-; Check vectorization on non-native interleaving factor 5
-define float @float_factor_5_deinterleave_cost(ptr %data, i64 %n) vscale_range(2, 0) {
-; CHECK-LABEL: define float @float_factor_5_deinterleave_cost(
+; Check that loops with non-native deinterleaving factors are not vectorized.
+; This patch prohibits vectorization with INTERLEAVE-GROUP, but the loop
+; is still vectorized via widening because vector inserts/extracts are currently
+; costed too cheaply for SVE. The insert/extract costs are addressed in a
+; subsequent patch, which should prevent this loop from being vectorized.
+
+define float @test_factor_5_deinterleaving(ptr %data, i64 %n) vscale_range(2, 0) {
+; CHECK-LABEL: define float @test_factor_5_deinterleaving(
; CHECK-SAME: ptr [[DATA:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], 16
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 3
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 15
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 16, i64 [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP3]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 1
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP5]], 2
-; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP5]], 3
-; CHECK-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP5]], 20
-; CHECK-NEXT: [[TMP10:%.*]] = mul nuw i64 [[TMP6]], 20
-; CHECK-NEXT: [[TMP7:%.*]] = mul nuw i64 [[TMP3]], 20
-; CHECK-NEXT: [[TMP8:%.*]] = mul nuw i64 [[TMP4]], 20
-; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP7]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[TMP36]], align 4
-; CHECK-NEXT: [[TMP14:%.*]] = load float, ptr [[TMP39]], align 4
-; CHECK-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[TMP12]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = fmul float 0.000000e+00, [[TMP13]]
-; CHECK-NEXT: [[TMP18:%.*]] = fmul float 0.000000e+00, [[TMP14]]
-; CHECK-NEXT: [[TMP19:%.*]] = fmul float 0.000000e+00, [[TMP15]]
-; CHECK-NEXT: [[TMP40:%.*]] = fmul float 0.000000e+00, [[TMP16]]
-; CHECK-NEXT: [[TMP21]] = call float @llvm.maxnum.f32(float [[VEC_PHI]], float [[TMP17]])
-; CHECK-NEXT: [[TMP22]] = call float @llvm.maxnum.f32(float [[VEC_PHI1]], float [[TMP18]])
-; CHECK-NEXT: [[TMP23]] = call float @llvm.maxnum.f32(float [[VEC_PHI2]], float [[TMP19]])
-; CHECK-NEXT: [[TMP24]] = call float @llvm.maxnum.f32(float [[VEC_PHI3]], float [[TMP40]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
-; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno float [[TMP17]], [[TMP18]]
-; CHECK-NEXT: [[TMP26:%.*]] = fcmp uno float [[TMP19]], [[TMP40]]
-; CHECK-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP25]]
-; CHECK-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP26]]
-; CHECK-NEXT: [[TMP29:%.*]] = or i1 [[TMP27]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP31:%.*]] = or i1 [[TMP29]], [[TMP30]]
-; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP72:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP73:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <8 x i64> [[VEC_IND]], splat (i64 8)
+; CHECK-NEXT: [[TMP4:%.*]] = mul nuw <8 x i64> [[VEC_IND]], splat (i64 20)
+; CHECK-NEXT: [[TMP5:%.*]] = mul nuw <8 x i64> [[STEP_ADD]], splat (i64 20)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i64> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i64> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i64> [[TMP4]], i64 2
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i64> [[TMP4]], i64 3
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i64> [[TMP4]], i64 4
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i64> [[TMP4]], i64 5
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i64> [[TMP4]], i64 6
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x i64> [[TMP4]], i64 7
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <8 x i64> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <8 x i64> [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <8 x i64> [[TMP5]], i64 2
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP26]]
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <8 x i64> [[TMP5]], i64 3
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP28]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <8 x i64> [[TMP5]], i64 4
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP30]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <8 x i64> [[TMP5]], i64 5
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP32]]
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <8 x i64> [[TMP5]], i64 6
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP34]]
+; CHECK-NEXT: [[TMP36:%.*]] = extractelement <8 x i64> [[TMP5]], i64 7
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = load float, ptr [[TMP7]], align 4
+; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP40:%.*]] = load float, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP41:%.*]] = load float, ptr [[TMP13]], align 4
+; CHECK-NEXT: [[TMP42:%.*]] = load float, ptr [[TMP15]], align 4
+; CHECK-NEXT: [[TMP43:%.*]] = load float, ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP44:%.*]] = load float, ptr [[TMP19]], align 4
+; CHECK-NEXT: [[TMP45:%.*]] = load float, ptr [[TMP21]], align 4
+; CHECK-NEXT: [[TMP46:%.*]] = insertelement <8 x float> poison, float [[TMP38]], i64 0
+; CHECK-NEXT: [[TMP47:%.*]] = insertelement <8 x float> [[TMP46]], float [[TMP39]], i64 1
+; CHECK-NEXT: [[TMP48:%.*]] = insertelement <8 x float> [[TMP47]], float [[TMP40]], i64 2
+; CHECK-NEXT: [[TMP49:%.*]] = insertelement <8 x float> [[TMP48]], float [[TMP41]], i64 3
+; CHECK-NEXT: [[TMP50:%.*]] = insertelement <8 x float> [[TMP49]], float [[TMP42]], i64 4
+; CHECK-NEXT: [[TMP51:%.*]] = insertelement <8 x float> [[TMP50]], float [[TMP43]], i64 5
+; CHECK-NEXT: [[TMP52:%.*]] = insertelement <8 x float> [[TMP51]], float [[TMP44]], i64 6
+; CHECK-NEXT: [[TMP53:%.*]] = insertelement <8 x float> [[TMP52]], float [[TMP45]], i64 7
+; CHECK-NEXT: [[TMP54:%.*]] = load float, ptr [[TMP23]], align 4
+; CHECK-NEXT: [[TMP55:%.*]] = load float, ptr [[TMP25]], align 4
+; CHECK-NEXT: [[TMP56:%.*]] = load float, ptr [[TMP27]], align 4
+; CHECK-NEXT: [[TMP57:%.*]] = load float, ptr [[TMP29]], align 4
+; CHECK-NEXT: [[TMP58:%.*]] = load float, ptr [[TMP31]], align 4
+; CHECK-NEXT: [[TMP59:%.*]] = load float, ptr [[TMP33]], align 4
+; CHECK-NEXT: [[TMP60:%.*]] = load float, ptr [[TMP35]], align 4
+; CHECK-NEXT: [[TMP61:%.*]] = load float, ptr [[TMP37]], align 4
+; CHECK-NEXT: [[TMP62:%.*]] = insertelement <8 x float> poison, float [[TMP54]], i64 0
+; CHECK-NEXT: [[TMP63:%.*]] = insertelement <8 x float> [[TMP62]], float [[TMP55]], i64 1
+; CHECK-NEXT: [[TMP64:%.*]] = insertelement <8 x float> [[TMP63]], float [[TMP56]], i64 2
+; CHECK-NEXT: [[TMP65:%.*]] = insertelement <8 x float> [[TMP64]], float [[TMP57]], i64 3
+; CHECK-NEXT: [[TMP66:%.*]] = insertelement <8 x float> [[TMP65]], float [[TMP58]], i64 4
+; CHECK-NEXT: [[TMP67:%.*]] = insertelement <8 x float> [[TMP66]], float [[TMP59]], i64 5
+; CHECK-NEXT: [[TMP68:%.*]] = insertelement <8 x float> [[TMP67]], float [[TMP60]], i64 6
+; CHECK-NEXT: [[TMP69:%.*]] = insertelement <8 x float> [[TMP68]], float [[TMP61]], i64 7
+; CHECK-NEXT: [[TMP70:%.*]] = fmul <8 x float> zeroinitializer, [[TMP53]]
+; CHECK-NEXT: [[TMP71:%.*]] = fmul <8 x float> zeroinitializer, [[TMP69]]
+; CHECK-NEXT: [[TMP72]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP70]])
+; CHECK-NEXT: [[TMP73]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI1]], <8 x float> [[TMP71]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP74:%.*]] = fcmp uno <8 x float> [[TMP70]], [[TMP71]]
+; CHECK-NEXT: [[TMP75:%.*]] = freeze <8 x i1> [[TMP74]]
+; CHECK-NEXT: [[TMP76:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP75]])
+; CHECK-NEXT: [[TMP77:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP78:%.*]] = or i1 [[TMP76]], [[TMP77]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[STEP_ADD]], splat (i64 8)
+; CHECK-NEXT: br i1 [[TMP78]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP32:%.*]] = select i1 [[TMP29]], float [[VEC_PHI]], float [[TMP21]]
-; CHECK-NEXT: [[TMP33:%.*]] = select i1 [[TMP29]], float [[VEC_PHI1]], float [[TMP22]]
-; CHECK-NEXT: [[TMP34:%.*]] = select i1 [[TMP29]], float [[VEC_PHI2]], float [[TMP23]]
-; CHECK-NEXT: [[TMP35:%.*]] = select i1 [[TMP29]], float [[VEC_PHI3]], float [[TMP24]]
-; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[TMP29]], i64 [[TMP5]], i64 [[N_VEC]]
-; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call float @llvm.maxnum.f32(float [[TMP32]], float [[TMP33]])
-; CHECK-NEXT: [[RDX_MINMAX4:%.*]] = call float @llvm.maxnum.f32(float [[RDX_MINMAX]], float [[TMP34]])
-; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call float @llvm.maxnum.f32(float [[RDX_MINMAX4]], float [[TMP35]])
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT: [[TMP37:%.*]] = xor i1 [[TMP29]], true
-; CHECK-NEXT: [[TMP38:%.*]] = and i1 [[CMP_N]], [[TMP37]]
-; CHECK-NEXT: br i1 [[TMP38]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP79:%.*]] = select i1 [[TMP76]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP72]]
+; CHECK-NEXT: [[TMP80:%.*]] = select i1 [[TMP76]], <8 x float> [[VEC_PHI1]], <8 x float> [[TMP73]]
+; CHECK-NEXT: [[TMP81:%.*]] = select i1 [[TMP76]], i64 [[INDEX]], i64 [[N_VEC]]
+; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[TMP79]], <8 x float> [[TMP80]])
+; CHECK-NEXT: [[TMP82:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[RDX_MINMAX]])
+; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP81]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP82]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -79,9 +126,9 @@ define float @float_factor_5_deinterleave_cost(ptr %data, i64 %n) vscale_range(2
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX_REC]], float [[PRODUCT]])
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[FOR_BODY]] ], [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[FOR_BODY]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -107,5 +154,5 @@ for.end: ; preds = %for.body
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
;.
>From 4bc4805e58d91875b584d90cd51674e77f1dcc08 Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Mon, 7 Sep 2026 16:08:26 +0000
Subject: [PATCH 4/8] Updated comments
---
.../AArch64/sve-non-native-interleaved-accesses.ll | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
index 026e5ccfc427f..da1f100f9f5e6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -2,9 +2,10 @@
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 \
; RUN: -S < %s | FileCheck %s
-; Check that loops with non-native deinterleaving factors are not vectorized.
-; This patch prohibits vectorization with INTERLEAVE-GROUP, but the loop
-; is still vectorized via widening because vector inserts/extracts are currently
+; Check that loops with non-native deinterleaving factors are not vectorized
+; using deinterleaving.
+; Vectorization is prohibited with INTERLEAVE-GROUP, but the loop
+; is still vectorized via scalarization because vector inserts/extracts are currently
; costed too cheaply for SVE. The insert/extract costs are addressed in a
; subsequent patch, which should prevent this loop from being vectorized.
>From 099ae04f02cebc670938624af110e026fd23a26b Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Wed, 2 Sep 2026 15:01:01 +0000
Subject: [PATCH 5/8] pre-commit test
---
.../AArch64/sve-vls-insert-extract.ll | 40 +++++++++++++++++++
1 file changed, 40 insertions(+)
create mode 100644 llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
new file mode 100644
index 0000000000000..9dbd8274758b8
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mattr=+sve | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define void @fixed_sve_high_lane_insert_extract(<8 x i32> %vi32, <8 x float> %vf32, <4 x double> %vf64, i32 %i32, float %f, double %d) vscale_range(2, 0) {
+; CHECK-LABEL: 'fixed_sve_high_lane_insert_extract'
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_elt0 = extractelement <8 x i32> %vi32, i32 0
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_low = extractelement <8 x i32> %vi32, i32 3
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_high = extractelement <8 x i32> %vi32, i32 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_elt0 = insertelement <8 x i32> %vi32, i32 %i32, i32 0
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_low = insertelement <8 x i32> %vi32, i32 %i32, i32 3
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_high = insertelement <8 x i32> %vi32, i32 %i32, i32 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_low = extractelement <8 x float> %vf32, i32 3
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_high = extractelement <8 x float> %vf32, i32 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef64_low = extractelement <4 x double> %vf64, i32 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef64_high = extractelement <4 x double> %vf64, i32 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_low = insertelement <8 x float> %vf32, float %f, i32 3
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_high = insertelement <8 x float> %vf32, float %f, i32 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if64_low = insertelement <4 x double> %vf64, double %d, i32 1
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if64_high = insertelement <4 x double> %vf64, double %d, i32 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %ei32_elt0 = extractelement <8 x i32> %vi32, i32 0
+ %ei32_low = extractelement <8 x i32> %vi32, i32 3
+ %ei32_high = extractelement <8 x i32> %vi32, i32 4
+ %ii32_elt0 = insertelement <8 x i32> %vi32, i32 %i32, i32 0
+ %ii32_low = insertelement <8 x i32> %vi32, i32 %i32, i32 3
+ %ii32_high = insertelement <8 x i32> %vi32, i32 %i32, i32 4
+ %ef32_low = extractelement <8 x float> %vf32, i32 3
+ %ef32_high = extractelement <8 x float> %vf32, i32 4
+ %ef64_low = extractelement <4 x double> %vf64, i32 1
+ %ef64_high = extractelement <4 x double> %vf64, i32 2
+ %if32_low = insertelement <8 x float> %vf32, float %f, i32 3
+ %if32_high = insertelement <8 x float> %vf32, float %f, i32 4
+ %if64_low = insertelement <4 x double> %vf64, double %d, i32 1
+ %if64_high = insertelement <4 x double> %vf64, double %d, i32 2
+ ret void
+}
>From 2bc0513f3fc0e6da93a0af4e753e13f98081039a Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Wed, 2 Sep 2026 15:06:06 +0000
Subject: [PATCH 6/8] [AArch64] Adressed review comments
Updated inserelement/extractelement costs:
- lanes within 128b, both cheap
- extracts from lanes >128b, float cheap, int needs FPR->GPR move
- inserts to lanes >128b is expensive
Updated tests
---
.../AArch64/AArch64TargetTransformInfo.cpp | 22 +++++
.../CostModel/AArch64/sve-intrinsics.ll | 4 +-
.../AArch64/sve-vls-insert-extract.ll | 8 +-
.../AArch64/sve-vls-shuffle-extract.ll | 80 +++++++++----------
4 files changed, 68 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 561067a3b0660..3107a093474d1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4599,6 +4599,28 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
if (Index == 0 && !Ty->getScalarType()->isIntegerTy())
return 0;
+ // SVE has no scalar move to an arbitrary lane above the low 128-bit portion
+ // of a Z register, e.g. there is no equivalent of "mov z0.d[9], d0".
+ // Fixed-length vectors wider than 128 bits therefore need
+ // [splice]/index/pred/splat/cmp/pred-mov when scalarizing inserts for those
+ // lanes, so model them as more expensive than ordinary NEON lane accesses.
+ // Extracts to integer types require extra mov from FPR -> GPR.
+ if (ST->useSVEForFixedLengthVectors() && isa<FixedVectorType>(Ty) &&
+ !Ty->isBFloatTy()) {
+ InstructionCost Cost = CostKind == TTI::TCK_CodeSize
+ ? 1
+ : ST->getVectorInsertExtractBaseCost();
+ unsigned LowElts = AArch64::SVEBitsPerBlock / Ty->getScalarSizeInBits();
+ if (Index >= LowElts) {
+ if (Opcode == Instruction::InsertElement)
+ Cost += 3;
+ else if (Opcode == Instruction::ExtractElement &&
+ Ty->getScalarType()->isIntegerTy())
+ Cost += 1;
+ }
+ return Cost;
+ }
+
// This is recognising a LD1 single-element structure to one lane of one
// register instruction. I.e., if this is an `insertelement` instruction,
// and its second operand is a load, then we will generate a LD1, which
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index aaa19cb4d4904..cf92e53f2dc70 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -103,7 +103,7 @@ define void @vector_insert_extract_idxzero_256b() #2 {
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> poison, <vscale x 2 x i1> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> poison, i64 0)
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> poison, <2 x float> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> poison, <vscale x 2 x float> poison, i64 0)
@@ -115,7 +115,7 @@ define void @vector_insert_extract_idxzero_256b() #2 {
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> poison, <vscale x 2 x i1> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> poison, i64 0)
-; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> poison, <2 x float> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> poison, <vscale x 2 x float> poison, i64 0)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
index 9dbd8274758b8..2fb66dfcbe5dc 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
@@ -8,18 +8,18 @@ define void @fixed_sve_high_lane_insert_extract(<8 x i32> %vi32, <8 x float> %vf
; CHECK-LABEL: 'fixed_sve_high_lane_insert_extract'
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_elt0 = extractelement <8 x i32> %vi32, i32 0
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_low = extractelement <8 x i32> %vi32, i32 3
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_high = extractelement <8 x i32> %vi32, i32 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:2 Lat:3 SizeLat:3 for: %ei32_high = extractelement <8 x i32> %vi32, i32 4
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_elt0 = insertelement <8 x i32> %vi32, i32 %i32, i32 0
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_low = insertelement <8 x i32> %vi32, i32 %i32, i32 3
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_high = insertelement <8 x i32> %vi32, i32 %i32, i32 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ii32_high = insertelement <8 x i32> %vi32, i32 %i32, i32 4
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_low = extractelement <8 x float> %vf32, i32 3
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_high = extractelement <8 x float> %vf32, i32 4
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef64_low = extractelement <4 x double> %vf64, i32 1
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef64_high = extractelement <4 x double> %vf64, i32 2
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_low = insertelement <8 x float> %vf32, float %f, i32 3
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_high = insertelement <8 x float> %vf32, float %f, i32 4
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if32_high = insertelement <8 x float> %vf32, float %f, i32 4
; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if64_low = insertelement <4 x double> %vf64, double %d, i32 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if64_high = insertelement <4 x double> %vf64, double %d, i32 2
+; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if64_high = insertelement <4 x double> %vf64, double %d, i32 2
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%ei32_elt0 = extractelement <8 x i32> %vi32, i32 0
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll
index 65261a8b2c6d6..8845454b8f015 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll
@@ -76,16 +76,16 @@ define void @extract_half_hi() {
; VSCALE-2-LABEL: 'extract_half_hi'
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> <i32 1>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-4-LABEL: 'extract_half_hi'
@@ -139,30 +139,30 @@ define void @extract_half_unaligned() {
;
; VSCALE-2-LABEL: 'extract_half_unaligned'
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 96 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 1, i32 2>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-4-LABEL: 'extract_half_unaligned'
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 200 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 100 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> <i32 1, i32 2>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> <i32 1, i32 2>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
%v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -233,30 +233,30 @@ define void @extract_qtr_hi() {
;
; VSCALE-2-LABEL: 'extract_qtr_hi'
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> <i32 4, i32 5>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> <i32 2>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 4, i32 5>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 2>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 2>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
; VSCALE-4-LABEL: 'extract_qtr_hi'
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> <i32 4, i32 5>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> <i32 2>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 2>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 2>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 4, i32 5>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
%v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
@@ -295,10 +295,10 @@ define void @extract_qtr_unaligned() {
; VSCALE-2-LABEL: 'extract_qtr_unaligned'
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> <i32 1>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -309,15 +309,15 @@ define void @extract_qtr_unaligned() {
; VSCALE-4-LABEL: 'extract_qtr_unaligned'
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> <i32 1>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> <i32 1>
-; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
+; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> <i32 2, i32 3>
; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
%v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
>From 35e7d7fd501f39cdffde630bac3fb6e44e91f830 Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Mon, 7 Sep 2026 11:07:08 +0000
Subject: [PATCH 7/8] Addressed review comments. - Added tests for bfloat -
Added tests for scalable vectors - Moved tests into sve-insert-extract.ll and
deleted sve-vls-insert-extract.ll - Updated LV test for non-native
deinterleaving factor
---
.../AArch64/AArch64TargetTransformInfo.cpp | 17 +-
.../CostModel/AArch64/sve-insert-extract.ll | 230 +++++++++++++++++-
.../CostModel/AArch64/sve-intrinsics.ll | 4 +-
.../AArch64/sve-vls-insert-extract.ll | 40 ---
.../sve-non-native-interleaved-accesses.ll | 133 ++++------
5 files changed, 284 insertions(+), 140 deletions(-)
delete mode 100644 llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 3107a093474d1..df7b20992de6b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4604,18 +4604,21 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
// Fixed-length vectors wider than 128 bits therefore need
// [splice]/index/pred/splat/cmp/pred-mov when scalarizing inserts for those
// lanes, so model them as more expensive than ordinary NEON lane accesses.
- // Extracts to integer types require extra mov from FPR -> GPR.
- if (ST->useSVEForFixedLengthVectors() && isa<FixedVectorType>(Ty) &&
- !Ty->isBFloatTy()) {
+ if (ST->useSVEForFixedLengthVectors()) {
InstructionCost Cost = CostKind == TTI::TCK_CodeSize
? 1
: ST->getVectorInsertExtractBaseCost();
unsigned LowElts = AArch64::SVEBitsPerBlock / Ty->getScalarSizeInBits();
- if (Index >= LowElts) {
- if (Opcode == Instruction::InsertElement)
+ if (Opcode == Instruction::InsertElement) {
+ if (Index >= LowElts)
Cost += 3;
- else if (Opcode == Instruction::ExtractElement &&
- Ty->getScalarType()->isIntegerTy())
+ } else if (Opcode == Instruction::ExtractElement) {
+ unsigned ExtRange = 4 * LowElts;
+ // Extract >512b needs mov-imm+whilels+lastb
+ if (Index >= ExtRange)
+ Cost += 2;
+ // Integer extracts (>128b, <512b) require extra mov from FPR -> GPR.
+ else if (Index >= LowElts && Ty->getScalarType()->isIntegerTy())
Cost += 1;
}
return Cost;
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll
index 5fcb1184045e3..3f5c88733f5c2 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-insert-extract.ll
@@ -1,8 +1,9 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --function fixed_sve_extract
; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -S < %s | FileCheck --check-prefix=CHECK-DEFAULT %s
; RUN: opt -aarch64-insert-extract-base-cost=0 -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -S < %s | FileCheck --check-prefix=CHECK-LOW %s
; RUN: opt -aarch64-insert-extract-base-cost=100000 -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -S < %s | FileCheck --check-prefix=CHECK-HIGH %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -aarch64-sve-vls-bfloat-support -S < %s | FileCheck --check-prefix=CHECK-VLS %s
target triple = "aarch64-unknown-linux-gnu"
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -218,6 +219,233 @@ define void @test_override_cpu_given() #1 {
+define void @fixed_sve_extract() #2 {
+; CHECK-DEFAULT-LABEL: 'fixed_sve_extract'
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_elt0 = extractelement <8 x i32> poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_low = extractelement <8 x i32> poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:3 CodeSize:2 Lat:3 SizeLat:3 for: %ei32_high = extractelement <8 x i32> poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0 = extractelement <8 x float> poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_low = extractelement <8 x float> poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_high = extractelement <8 x float> poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0 = extractelement <16 x bfloat> poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ebf16_low = extractelement <16 x bfloat> poison, i32 7
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ebf16_high = extractelement <16 x bfloat> poison, i32 8
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_elt0_sv = extractelement <vscale x 4 x i32> poison, i64 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_low_sv = extractelement <vscale x 4 x i32> poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:3 CodeSize:2 Lat:3 SizeLat:3 for: %ei32_high_sv = extractelement <vscale x 4 x i32> poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %ei32_512b_sv = extractelement <vscale x 4 x i32> poison, i32 16
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0_sv = extractelement <vscale x 4 x float> poison, i64 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_low_sv = extractelement <vscale x 4 x float> poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_high_sv = extractelement <vscale x 4 x float> poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %ef32_512b_sv = extractelement <vscale x 4 x float> poison, i32 16
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0_sv = extractelement <vscale x 8 x bfloat> poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ebf16_low_sv = extractelement <vscale x 8 x bfloat> poison, i32 7
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ebf16_high_sv = extractelement <vscale x 8 x bfloat> poison, i32 8
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %ebf16_512b_sv = extractelement <vscale x 8 x bfloat> poison, i32 32
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-LOW-LABEL: 'fixed_sve_extract'
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ei32_elt0 = extractelement <8 x i32> poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ei32_low = extractelement <8 x i32> poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %ei32_high = extractelement <8 x i32> poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0 = extractelement <8 x float> poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ef32_low = extractelement <8 x float> poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ef32_high = extractelement <8 x float> poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0 = extractelement <16 x bfloat> poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ebf16_low = extractelement <16 x bfloat> poison, i32 7
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ebf16_high = extractelement <16 x bfloat> poison, i32 8
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ei32_elt0_sv = extractelement <vscale x 4 x i32> poison, i64 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ei32_low_sv = extractelement <vscale x 4 x i32> poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %ei32_high_sv = extractelement <vscale x 4 x i32> poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %ei32_512b_sv = extractelement <vscale x 4 x i32> poison, i32 16
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0_sv = extractelement <vscale x 4 x float> poison, i64 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ef32_low_sv = extractelement <vscale x 4 x float> poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ef32_high_sv = extractelement <vscale x 4 x float> poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %ef32_512b_sv = extractelement <vscale x 4 x float> poison, i32 16
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0_sv = extractelement <vscale x 8 x bfloat> poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ebf16_low_sv = extractelement <vscale x 8 x bfloat> poison, i32 7
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ebf16_high_sv = extractelement <vscale x 8 x bfloat> poison, i32 8
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:2 for: %ebf16_512b_sv = extractelement <vscale x 8 x bfloat> poison, i32 32
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'fixed_sve_extract'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ei32_elt0 = extractelement <8 x i32> poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ei32_low = extractelement <8 x i32> poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100001 CodeSize:2 Lat:100001 SizeLat:100001 for: %ei32_high = extractelement <8 x i32> poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0 = extractelement <8 x float> poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ef32_low = extractelement <8 x float> poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ef32_high = extractelement <8 x float> poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0 = extractelement <16 x bfloat> poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ebf16_low = extractelement <16 x bfloat> poison, i32 7
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ebf16_high = extractelement <16 x bfloat> poison, i32 8
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ei32_elt0_sv = extractelement <vscale x 4 x i32> poison, i64 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ei32_low_sv = extractelement <vscale x 4 x i32> poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100001 CodeSize:2 Lat:100001 SizeLat:100001 for: %ei32_high_sv = extractelement <vscale x 4 x i32> poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100002 CodeSize:3 Lat:100002 SizeLat:100002 for: %ei32_512b_sv = extractelement <vscale x 4 x i32> poison, i32 16
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0_sv = extractelement <vscale x 4 x float> poison, i64 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ef32_low_sv = extractelement <vscale x 4 x float> poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ef32_high_sv = extractelement <vscale x 4 x float> poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100002 CodeSize:3 Lat:100002 SizeLat:100002 for: %ef32_512b_sv = extractelement <vscale x 4 x float> poison, i32 16
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0_sv = extractelement <vscale x 8 x bfloat> poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ebf16_low_sv = extractelement <vscale x 8 x bfloat> poison, i32 7
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ebf16_high_sv = extractelement <vscale x 8 x bfloat> poison, i32 8
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100002 CodeSize:3 Lat:100002 SizeLat:100002 for: %ebf16_512b_sv = extractelement <vscale x 8 x bfloat> poison, i32 32
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-VLS-LABEL: 'fixed_sve_extract'
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_elt0 = extractelement <8 x i32> poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_low = extractelement <8 x i32> poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:3 CodeSize:2 Lat:3 SizeLat:3 for: %ei32_high = extractelement <8 x i32> poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0 = extractelement <8 x float> poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_low = extractelement <8 x float> poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_high = extractelement <8 x float> poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0 = extractelement <16 x bfloat> poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ebf16_low = extractelement <16 x bfloat> poison, i32 7
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ebf16_high = extractelement <16 x bfloat> poison, i32 8
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_elt0_sv = extractelement <vscale x 4 x i32> poison, i64 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_low_sv = extractelement <vscale x 4 x i32> poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:3 CodeSize:2 Lat:3 SizeLat:3 for: %ei32_high_sv = extractelement <vscale x 4 x i32> poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %ei32_512b_sv = extractelement <vscale x 4 x i32> poison, i32 16
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %ef32_elt0_sv = extractelement <vscale x 4 x float> poison, i64 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_low_sv = extractelement <vscale x 4 x float> poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_high_sv = extractelement <vscale x 4 x float> poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %ef32_512b_sv = extractelement <vscale x 4 x float> poison, i32 16
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %ebf16_elt0_sv = extractelement <vscale x 8 x bfloat> poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ebf16_low_sv = extractelement <vscale x 8 x bfloat> poison, i32 7
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ebf16_high_sv = extractelement <vscale x 8 x bfloat> poison, i32 8
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %ebf16_512b_sv = extractelement <vscale x 8 x bfloat> poison, i32 32
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %ei32_elt0 = extractelement <8 x i32> poison, i32 0
+ %ei32_low = extractelement <8 x i32> poison, i32 3
+ %ei32_high = extractelement <8 x i32> poison, i32 4
+ %ef32_elt0 = extractelement <8 x float> poison, i32 0
+ %ef32_low = extractelement <8 x float> poison, i32 3
+ %ef32_high = extractelement <8 x float> poison, i32 4
+ %ebf16_elt0 = extractelement <16 x bfloat> poison, i32 0
+ %ebf16_low = extractelement <16 x bfloat> poison, i32 7
+ %ebf16_high = extractelement <16 x bfloat> poison, i32 8
+ %ei32_elt0_sv = extractelement <vscale x 4 x i32> poison, i64 0
+ %ei32_low_sv = extractelement <vscale x 4 x i32> poison, i32 3
+ %ei32_high_sv = extractelement <vscale x 4 x i32> poison, i32 4
+ %ei32_512b_sv = extractelement <vscale x 4 x i32> poison, i32 16
+ %ef32_elt0_sv = extractelement <vscale x 4 x float> poison, i64 0
+ %ef32_low_sv = extractelement <vscale x 4 x float> poison, i32 3
+ %ef32_high_sv = extractelement <vscale x 4 x float> poison, i32 4
+ %ef32_512b_sv = extractelement <vscale x 4 x float> poison, i32 16
+ %ebf16_elt0_sv = extractelement <vscale x 8 x bfloat> poison, i32 0
+ %ebf16_low_sv = extractelement <vscale x 8 x bfloat> poison, i32 7
+ %ebf16_high_sv = extractelement <vscale x 8 x bfloat> poison, i32 8
+ %ebf16_512b_sv = extractelement <vscale x 8 x bfloat> poison, i32 32
+ ret void
+}
+
+define void @fixed_sve_insert() #2 {
+; CHECK-DEFAULT-LABEL: 'fixed_sve_insert'
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_elt0 = insertelement <8 x i32> poison, i32 poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_low = insertelement <8 x i32> poison, i32 poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ii32_high = insertelement <8 x i32> poison, i32 poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %if32_elt0 = insertelement <8 x float> poison, float poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_low = insertelement <8 x float> poison, float poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if32_high = insertelement <8 x float> poison, float poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0 = insertelement <16 x bfloat> poison, bfloat poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ibf16_low = insertelement <16 x bfloat> poison, bfloat poison, i32 7
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ibf16_high = insertelement <16 x bfloat> poison, bfloat poison, i32 8
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_elt0_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i64 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_low_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ii32_high_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %if32_elt0_sv = insertelement <vscale x 4 x float> poison, float poison, i64 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_low_sv = insertelement <vscale x 4 x float> poison, float poison, i32 3
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if32_high_sv = insertelement <vscale x 4 x float> poison, float poison, i32 4
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 0
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ibf16_low_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 7
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ibf16_high_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 8
+; CHECK-DEFAULT-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-LOW-LABEL: 'fixed_sve_insert'
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ii32_elt0 = insertelement <8 x i32> poison, i32 poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ii32_low = insertelement <8 x i32> poison, i32 poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %ii32_high = insertelement <8 x i32> poison, i32 poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %if32_elt0 = insertelement <8 x float> poison, float poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %if32_low = insertelement <8 x float> poison, float poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %if32_high = insertelement <8 x float> poison, float poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0 = insertelement <16 x bfloat> poison, bfloat poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ibf16_low = insertelement <16 x bfloat> poison, bfloat poison, i32 7
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ibf16_high = insertelement <16 x bfloat> poison, bfloat poison, i32 8
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ii32_elt0_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i64 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ii32_low_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %ii32_high_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %if32_elt0_sv = insertelement <vscale x 4 x float> poison, float poison, i64 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %if32_low_sv = insertelement <vscale x 4 x float> poison, float poison, i32 3
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %if32_high_sv = insertelement <vscale x 4 x float> poison, float poison, i32 4
+; CHECK-LOW-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 0
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:0 SizeLat:0 for: %ibf16_low_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 7
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:3 SizeLat:3 for: %ibf16_high_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 8
+; CHECK-LOW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-HIGH-LABEL: 'fixed_sve_insert'
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ii32_elt0 = insertelement <8 x i32> poison, i32 poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ii32_low = insertelement <8 x i32> poison, i32 poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100003 CodeSize:4 Lat:100003 SizeLat:100003 for: %ii32_high = insertelement <8 x i32> poison, i32 poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %if32_elt0 = insertelement <8 x float> poison, float poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %if32_low = insertelement <8 x float> poison, float poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100003 CodeSize:4 Lat:100003 SizeLat:100003 for: %if32_high = insertelement <8 x float> poison, float poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0 = insertelement <16 x bfloat> poison, bfloat poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ibf16_low = insertelement <16 x bfloat> poison, bfloat poison, i32 7
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ibf16_high = insertelement <16 x bfloat> poison, bfloat poison, i32 8
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ii32_elt0_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i64 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ii32_low_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100003 CodeSize:4 Lat:100003 SizeLat:100003 for: %ii32_high_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %if32_elt0_sv = insertelement <vscale x 4 x float> poison, float poison, i64 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %if32_low_sv = insertelement <vscale x 4 x float> poison, float poison, i32 3
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100003 CodeSize:4 Lat:100003 SizeLat:100003 for: %if32_high_sv = insertelement <vscale x 4 x float> poison, float poison, i32 4
+; CHECK-HIGH-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 0
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100000 CodeSize:1 Lat:100000 SizeLat:100000 for: %ibf16_low_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 7
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:100003 CodeSize:4 Lat:100003 SizeLat:100003 for: %ibf16_high_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 8
+; CHECK-HIGH-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-VLS-LABEL: 'fixed_sve_insert'
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_elt0 = insertelement <8 x i32> poison, i32 poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_low = insertelement <8 x i32> poison, i32 poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ii32_high = insertelement <8 x i32> poison, i32 poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %if32_elt0 = insertelement <8 x float> poison, float poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_low = insertelement <8 x float> poison, float poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if32_high = insertelement <8 x float> poison, float poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0 = insertelement <16 x bfloat> poison, bfloat poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ibf16_low = insertelement <16 x bfloat> poison, bfloat poison, i32 7
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ibf16_high = insertelement <16 x bfloat> poison, bfloat poison, i32 8
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_elt0_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i64 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_low_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ii32_high_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %if32_elt0_sv = insertelement <vscale x 4 x float> poison, float poison, i64 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_low_sv = insertelement <vscale x 4 x float> poison, float poison, i32 3
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if32_high_sv = insertelement <vscale x 4 x float> poison, float poison, i32 4
+; CHECK-VLS-NEXT: Cost Model: Found costs of 0 for: %ibf16_elt0_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 0
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ibf16_low_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 7
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ibf16_high_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 8
+; CHECK-VLS-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %ii32_elt0 = insertelement <8 x i32> poison, i32 poison, i32 0
+ %ii32_low = insertelement <8 x i32> poison, i32 poison, i32 3
+ %ii32_high = insertelement <8 x i32> poison, i32 poison, i32 4
+ %if32_elt0 = insertelement <8 x float> poison, float poison, i32 0
+ %if32_low = insertelement <8 x float> poison, float poison, i32 3
+ %if32_high = insertelement <8 x float> poison, float poison, i32 4
+ %ibf16_elt0 = insertelement <16 x bfloat> poison, bfloat poison, i32 0
+ %ibf16_low = insertelement <16 x bfloat> poison, bfloat poison, i32 7
+ %ibf16_high = insertelement <16 x bfloat> poison, bfloat poison, i32 8
+ %ii32_elt0_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i64 0
+ %ii32_low_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 3
+ %ii32_high_sv = insertelement <vscale x 4 x i32> poison, i32 poison, i32 4
+ %if32_elt0_sv = insertelement <vscale x 4 x float> poison, float poison, i64 0
+ %if32_low_sv = insertelement <vscale x 4 x float> poison, float poison, i32 3
+ %if32_high_sv = insertelement <vscale x 4 x float> poison, float poison, i32 4
+ %ibf16_elt0_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 0
+ %ibf16_low_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 7
+ %ibf16_high_sv = insertelement <vscale x 8 x bfloat> poison, bfloat poison, i32 8
+ ret void
+}
attributes #0 = { "target-features"="+sve" vscale_range(1, 16) }
attributes #1 = { "target-features"="+sve" vscale_range(1, 16) "target-cpu"="kryo" }
+attributes #2 = { "target-features"="+sve" vscale_range(2, 0) }
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index cf92e53f2dc70..40305e360ab2c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -103,7 +103,7 @@ define void @vector_insert_extract_idxzero_256b() #2 {
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> poison, <vscale x 2 x i1> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> poison, i64 0)
-; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
+; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:32 CodeSize:16 Lat:32 SizeLat:32 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> poison, <2 x float> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> poison, i64 0)
; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> poison, <vscale x 2 x float> poison, i64 0)
@@ -115,7 +115,7 @@ define void @vector_insert_extract_idxzero_256b() #2 {
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 0 for: %extract_legal_fixed_from_scalable = call <8 x float> @llvm.vector.extract.v8f32.nxv4f32(<vscale x 4 x float> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %insert_nxv16i1_nxv2i1 = call <vscale x 16 x i1> @llvm.vector.insert.nxv16i1.nxv2i1(<vscale x 16 x i1> poison, <vscale x 2 x i1> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %extract_nxv4i1_nxv16i1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv16i1(<vscale x 16 x i1> poison, i64 0)
-; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:24 Lat:40 SizeLat:40 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
+; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:32 CodeSize:16 Lat:32 SizeLat:32 for: %extract_v8i1_nxv8i1 = call <8 x i1> @llvm.vector.extract.v8i1.nxv8i1(<vscale x 8 x i1> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %insert_v2f32_nxv2f32 = call <vscale x 2 x float> @llvm.vector.insert.nxv2f32.v2f32(<vscale x 2 x float> poison, <2 x float> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %extract_v4f16_nxv4f16 = call <4 x half> @llvm.vector.extract.v4f16.nxv4f16(<vscale x 4 x half> poison, i64 0)
; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %insert_nxv2f32_nxv4f32 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.nxv2f32(<vscale x 4 x float> poison, <vscale x 2 x float> poison, i64 0)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
deleted file mode 100644
index 2fb66dfcbe5dc..0000000000000
--- a/llvm/test/Analysis/CostModel/AArch64/sve-vls-insert-extract.ll
+++ /dev/null
@@ -1,40 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mattr=+sve | FileCheck %s
-
-target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-gnu"
-
-define void @fixed_sve_high_lane_insert_extract(<8 x i32> %vi32, <8 x float> %vf32, <4 x double> %vf64, i32 %i32, float %f, double %d) vscale_range(2, 0) {
-; CHECK-LABEL: 'fixed_sve_high_lane_insert_extract'
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_elt0 = extractelement <8 x i32> %vi32, i32 0
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ei32_low = extractelement <8 x i32> %vi32, i32 3
-; CHECK-NEXT: Cost Model: Found costs of RThru:3 CodeSize:2 Lat:3 SizeLat:3 for: %ei32_high = extractelement <8 x i32> %vi32, i32 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_elt0 = insertelement <8 x i32> %vi32, i32 %i32, i32 0
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ii32_low = insertelement <8 x i32> %vi32, i32 %i32, i32 3
-; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %ii32_high = insertelement <8 x i32> %vi32, i32 %i32, i32 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_low = extractelement <8 x float> %vf32, i32 3
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef32_high = extractelement <8 x float> %vf32, i32 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef64_low = extractelement <4 x double> %vf64, i32 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %ef64_high = extractelement <4 x double> %vf64, i32 2
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if32_low = insertelement <8 x float> %vf32, float %f, i32 3
-; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if32_high = insertelement <8 x float> %vf32, float %f, i32 4
-; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %if64_low = insertelement <4 x double> %vf64, double %d, i32 1
-; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:5 SizeLat:5 for: %if64_high = insertelement <4 x double> %vf64, double %d, i32 2
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
- %ei32_elt0 = extractelement <8 x i32> %vi32, i32 0
- %ei32_low = extractelement <8 x i32> %vi32, i32 3
- %ei32_high = extractelement <8 x i32> %vi32, i32 4
- %ii32_elt0 = insertelement <8 x i32> %vi32, i32 %i32, i32 0
- %ii32_low = insertelement <8 x i32> %vi32, i32 %i32, i32 3
- %ii32_high = insertelement <8 x i32> %vi32, i32 %i32, i32 4
- %ef32_low = extractelement <8 x float> %vf32, i32 3
- %ef32_high = extractelement <8 x float> %vf32, i32 4
- %ef64_low = extractelement <4 x double> %vf64, i32 1
- %ef64_high = extractelement <4 x double> %vf64, i32 2
- %if32_low = insertelement <8 x float> %vf32, float %f, i32 3
- %if32_high = insertelement <8 x float> %vf32, float %f, i32 4
- %if64_low = insertelement <4 x double> %vf64, double %d, i32 1
- %if64_high = insertelement <4 x double> %vf64, double %d, i32 2
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
index da1f100f9f5e6..65ac9fda09c33 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-non-native-interleaved-accesses.ll
@@ -2,120 +2,73 @@
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown-linux-gnu -mattr=+sve2 \
; RUN: -S < %s | FileCheck %s
-; Check that loops with non-native deinterleaving factors are not vectorized
-; using deinterleaving.
-; Vectorization is prohibited with INTERLEAVE-GROUP, but the loop
-; is still vectorized via scalarization because vector inserts/extracts are currently
-; costed too cheaply for SVE. The insert/extract costs are addressed in a
-; subsequent patch, which should prevent this loop from being vectorized.
+; Check that loops with non-native deinterleaving factors are not vectorized.
define float @test_factor_5_deinterleaving(ptr %data, i64 %n) vscale_range(2, 0) {
; CHECK-LABEL: define float @test_factor_5_deinterleaving(
; CHECK-SAME: ptr [[DATA:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP0]], 16
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 15
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 16, i64 [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP0]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[TMP3]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP72:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <8 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP73:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <8 x i64> [[VEC_IND]], splat (i64 8)
-; CHECK-NEXT: [[TMP4:%.*]] = mul nuw <8 x i64> [[VEC_IND]], splat (i64 20)
-; CHECK-NEXT: [[TMP5:%.*]] = mul nuw <8 x i64> [[STEP_ADD]], splat (i64 20)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i64> [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i64> [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i64> [[TMP4]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i64> [[TMP4]], i64 3
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i64> [[TMP4]], i64 4
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i64> [[TMP4]], i64 5
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i64> [[TMP4]], i64 6
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x i64> [[TMP4]], i64 7
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP20]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <8 x i64> [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <8 x i64> [[TMP5]], i64 1
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP24]]
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <8 x i64> [[TMP5]], i64 2
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP26]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <8 x i64> [[TMP5]], i64 3
-; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <8 x i64> [[TMP5]], i64 4
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP30:%.*]] = mul nuw i64 [[INDEX]], 20
+; CHECK-NEXT: [[TMP32:%.*]] = mul nuw i64 [[TMP2]], 20
+; CHECK-NEXT: [[TMP34:%.*]] = mul nuw i64 [[TMP5]], 20
+; CHECK-NEXT: [[TMP36:%.*]] = mul nuw i64 [[TMP4]], 20
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP30]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <8 x i64> [[TMP5]], i64 5
; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP32]]
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <8 x i64> [[TMP5]], i64 6
; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = extractelement <8 x i64> [[TMP5]], i64 7
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = load float, ptr [[TMP7]], align 4
-; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[TMP9]], align 4
-; CHECK-NEXT: [[TMP40:%.*]] = load float, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP41:%.*]] = load float, ptr [[TMP13]], align 4
-; CHECK-NEXT: [[TMP42:%.*]] = load float, ptr [[TMP15]], align 4
-; CHECK-NEXT: [[TMP43:%.*]] = load float, ptr [[TMP17]], align 4
-; CHECK-NEXT: [[TMP44:%.*]] = load float, ptr [[TMP19]], align 4
-; CHECK-NEXT: [[TMP45:%.*]] = load float, ptr [[TMP21]], align 4
-; CHECK-NEXT: [[TMP46:%.*]] = insertelement <8 x float> poison, float [[TMP38]], i64 0
-; CHECK-NEXT: [[TMP47:%.*]] = insertelement <8 x float> [[TMP46]], float [[TMP39]], i64 1
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <8 x float> [[TMP47]], float [[TMP40]], i64 2
-; CHECK-NEXT: [[TMP49:%.*]] = insertelement <8 x float> [[TMP48]], float [[TMP41]], i64 3
-; CHECK-NEXT: [[TMP50:%.*]] = insertelement <8 x float> [[TMP49]], float [[TMP42]], i64 4
-; CHECK-NEXT: [[TMP51:%.*]] = insertelement <8 x float> [[TMP50]], float [[TMP43]], i64 5
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <8 x float> [[TMP51]], float [[TMP44]], i64 6
-; CHECK-NEXT: [[TMP53:%.*]] = insertelement <8 x float> [[TMP52]], float [[TMP45]], i64 7
-; CHECK-NEXT: [[TMP54:%.*]] = load float, ptr [[TMP23]], align 4
-; CHECK-NEXT: [[TMP55:%.*]] = load float, ptr [[TMP25]], align 4
-; CHECK-NEXT: [[TMP56:%.*]] = load float, ptr [[TMP27]], align 4
-; CHECK-NEXT: [[TMP57:%.*]] = load float, ptr [[TMP29]], align 4
; CHECK-NEXT: [[TMP58:%.*]] = load float, ptr [[TMP31]], align 4
; CHECK-NEXT: [[TMP59:%.*]] = load float, ptr [[TMP33]], align 4
; CHECK-NEXT: [[TMP60:%.*]] = load float, ptr [[TMP35]], align 4
; CHECK-NEXT: [[TMP61:%.*]] = load float, ptr [[TMP37]], align 4
-; CHECK-NEXT: [[TMP62:%.*]] = insertelement <8 x float> poison, float [[TMP54]], i64 0
-; CHECK-NEXT: [[TMP63:%.*]] = insertelement <8 x float> [[TMP62]], float [[TMP55]], i64 1
-; CHECK-NEXT: [[TMP64:%.*]] = insertelement <8 x float> [[TMP63]], float [[TMP56]], i64 2
-; CHECK-NEXT: [[TMP65:%.*]] = insertelement <8 x float> [[TMP64]], float [[TMP57]], i64 3
-; CHECK-NEXT: [[TMP66:%.*]] = insertelement <8 x float> [[TMP65]], float [[TMP58]], i64 4
-; CHECK-NEXT: [[TMP67:%.*]] = insertelement <8 x float> [[TMP66]], float [[TMP59]], i64 5
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <8 x float> [[TMP67]], float [[TMP60]], i64 6
-; CHECK-NEXT: [[TMP69:%.*]] = insertelement <8 x float> [[TMP68]], float [[TMP61]], i64 7
-; CHECK-NEXT: [[TMP70:%.*]] = fmul <8 x float> zeroinitializer, [[TMP53]]
-; CHECK-NEXT: [[TMP71:%.*]] = fmul <8 x float> zeroinitializer, [[TMP69]]
-; CHECK-NEXT: [[TMP72]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI]], <8 x float> [[TMP70]])
-; CHECK-NEXT: [[TMP73]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VEC_PHI1]], <8 x float> [[TMP71]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT: [[TMP74:%.*]] = fcmp uno <8 x float> [[TMP70]], [[TMP71]]
-; CHECK-NEXT: [[TMP75:%.*]] = freeze <8 x i1> [[TMP74]]
-; CHECK-NEXT: [[TMP76:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP75]])
+; CHECK-NEXT: [[TMP17:%.*]] = fmul float 0.000000e+00, [[TMP58]]
+; CHECK-NEXT: [[TMP18:%.*]] = fmul float 0.000000e+00, [[TMP59]]
+; CHECK-NEXT: [[TMP19:%.*]] = fmul float 0.000000e+00, [[TMP60]]
+; CHECK-NEXT: [[TMP20:%.*]] = fmul float 0.000000e+00, [[TMP61]]
+; CHECK-NEXT: [[TMP21]] = call float @llvm.maxnum.f32(float [[VEC_PHI]], float [[TMP17]])
+; CHECK-NEXT: [[TMP22]] = call float @llvm.maxnum.f32(float [[VEC_PHI1]], float [[TMP18]])
+; CHECK-NEXT: [[TMP23]] = call float @llvm.maxnum.f32(float [[VEC_PHI2]], float [[TMP19]])
+; CHECK-NEXT: [[TMP24]] = call float @llvm.maxnum.f32(float [[VEC_PHI3]], float [[TMP20]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP25:%.*]] = fcmp uno float [[TMP17]], [[TMP18]]
+; CHECK-NEXT: [[TMP26:%.*]] = fcmp uno float [[TMP19]], [[TMP20]]
+; CHECK-NEXT: [[TMP27:%.*]] = freeze i1 [[TMP25]]
+; CHECK-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP26]]
+; CHECK-NEXT: [[TMP76:%.*]] = or i1 [[TMP27]], [[TMP28]]
; CHECK-NEXT: [[TMP77:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP78:%.*]] = or i1 [[TMP76]], [[TMP77]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[STEP_ADD]], splat (i64 8)
; CHECK-NEXT: br i1 [[TMP78]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP79:%.*]] = select i1 [[TMP76]], <8 x float> [[VEC_PHI]], <8 x float> [[TMP72]]
-; CHECK-NEXT: [[TMP80:%.*]] = select i1 [[TMP76]], <8 x float> [[VEC_PHI1]], <8 x float> [[TMP73]]
+; CHECK-NEXT: [[TMP39:%.*]] = select i1 [[TMP76]], float [[VEC_PHI]], float [[TMP21]]
+; CHECK-NEXT: [[TMP40:%.*]] = select i1 [[TMP76]], float [[VEC_PHI1]], float [[TMP22]]
+; CHECK-NEXT: [[TMP41:%.*]] = select i1 [[TMP76]], float [[VEC_PHI2]], float [[TMP23]]
+; CHECK-NEXT: [[TMP42:%.*]] = select i1 [[TMP76]], float [[VEC_PHI3]], float [[TMP24]]
; CHECK-NEXT: [[TMP81:%.*]] = select i1 [[TMP76]], i64 [[INDEX]], i64 [[N_VEC]]
-; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[TMP79]], <8 x float> [[TMP80]])
-; CHECK-NEXT: [[TMP82:%.*]] = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> [[RDX_MINMAX]])
-; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK-NEXT: [[RDX_MINMAX:%.*]] = call float @llvm.maxnum.f32(float [[TMP39]], float [[TMP40]])
+; CHECK-NEXT: [[RDX_MINMAX4:%.*]] = call float @llvm.maxnum.f32(float [[RDX_MINMAX]], float [[TMP41]])
+; CHECK-NEXT: [[RDX_MINMAX5:%.*]] = call float @llvm.maxnum.f32(float [[RDX_MINMAX4]], float [[TMP42]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT: [[TMP43:%.*]] = xor i1 [[TMP76]], true
+; CHECK-NEXT: [[TMP38:%.*]] = and i1 [[CMP_N]], [[TMP43]]
+; CHECK-NEXT: br i1 [[TMP38]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP81]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP82]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
@@ -127,9 +80,9 @@ define float @test_factor_5_deinterleaving(ptr %data, i64 %n) vscale_range(2, 0)
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX_REC]], float [[PRODUCT]])
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[FOR_END]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[FOR_BODY]] ], [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -155,5 +108,5 @@ for.end: ; preds = %for.body
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
;.
>From 33a0ad21e218d5789d1a31b79ec49394912e7dbb Mon Sep 17 00:00:00 2001
From: Utpal Bora <utpal.bora2 at arm.com>
Date: Mon, 7 Sep 2026 17:25:21 +0000
Subject: [PATCH 8/8] Code rewrite for nested ifs
---
.../AArch64/AArch64TargetTransformInfo.cpp | 23 ++++++++-----------
1 file changed, 10 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index df7b20992de6b..8e5846cd41b63 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4608,20 +4608,17 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
InstructionCost Cost = CostKind == TTI::TCK_CodeSize
? 1
: ST->getVectorInsertExtractBaseCost();
- unsigned LowElts = AArch64::SVEBitsPerBlock / Ty->getScalarSizeInBits();
- if (Opcode == Instruction::InsertElement) {
- if (Index >= LowElts)
- Cost += 3;
- } else if (Opcode == Instruction::ExtractElement) {
- unsigned ExtRange = 4 * LowElts;
- // Extract >512b needs mov-imm+whilels+lastb
- if (Index >= ExtRange)
- Cost += 2;
+ if (Index * Ty->getScalarSizeInBits() < 128)
+ return Cost;
+ if (Index * Ty->getScalarSizeInBits() < 512 &&
+ Opcode == Instruction::ExtractElement)
// Integer extracts (>128b, <512b) require extra mov from FPR -> GPR.
- else if (Index >= LowElts && Ty->getScalarType()->isIntegerTy())
- Cost += 1;
- }
- return Cost;
+ return Ty->getScalarType()->isIntegerTy() ? Cost + 1 : Cost;
+ if (Opcode == Instruction::ExtractElement)
+ return Cost + 2; // cost of mov imm + whilels + lastb
+ if (Opcode == Instruction::InsertElement)
+ return Cost + 3; // cost of insert with cmp/splice
+ llvm_unreachable("unexpected opcode");
}
// This is recognising a LD1 single-element structure to one lane of one
More information about the llvm-commits
mailing list