[llvm] [AArch64] Change the default MaxInterleaveFactor to 4 for scalar loops. (PR #201611)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 4 08:25:31 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/201611
The default is 2, many CPUs already have this value set to 4, so the effect of this is to cause some loops that were interleaved 2x to be interleaved 4x when no vectorization occurs. This should help small inorder CPUs that like scheduling and larger out of order cpus that tend to have more pipelines to execute multiple instructions per cycle.
>From 326e91b508f989ed6d317ce3fafcc228acf54eb7 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Thu, 4 Jun 2026 16:17:26 +0100
Subject: [PATCH] [AArch64] Change the default MaxInterleaveFactor to 4 for
scalar loops.
The default is 2, many CPUs already have this value set to 4, so the effect of
this is to cause some loops that were interleaved 2x to be interleaved 4x when
no vectorization occurs. This should help small inorder CPUs that like
scheduling and larger out of order cpus that tend to have more pipelines to
execute multiple instructions per cycle.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 2 +
.../AArch64/induction-costs-sve.ll | 64 +++++++++++++++----
.../LoopVectorize/AArch64/induction-costs.ll | 12 +++-
.../AArch64/partial-reduce-dot-product.ll | 34 +++++++++-
.../AArch64/replicating-load-store-costs.ll | 18 +++++-
.../LoopVectorize/AArch64/sve-illegal-type.ll | 16 +++++
6 files changed, 125 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 730bec428a38e..e1a5b609e88ce 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5189,6 +5189,8 @@ bool AArch64TTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
}
unsigned AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF) const {
+ if (VF.isScalar())
+ return 4;
return ST->getMaxInterleaveFactor();
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 8aca5b6022661..3675576d6bbf4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -204,7 +204,7 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[ENTRY:.*]]:
; DEFAULT-NEXT: [[MUL_X:%.*]] = add i32 [[X]], 1
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; DEFAULT: [[VECTOR_SCEVCHECK]]:
; DEFAULT-NEXT: [[TMP1:%.*]] = sub i32 -1, [[X]]
@@ -224,22 +224,32 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
; DEFAULT-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP13:%.*]] = trunc i64 [[INDEX]] to i32
; DEFAULT-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 1
+; DEFAULT-NEXT: [[TMP22:%.*]] = add i32 [[TMP13]], 2
+; DEFAULT-NEXT: [[TMP25:%.*]] = add i32 [[TMP13]], 3
; DEFAULT-NEXT: [[TMP15:%.*]] = mul i32 [[MUL_X]], [[TMP13]]
; DEFAULT-NEXT: [[TMP16:%.*]] = mul i32 [[MUL_X]], [[TMP14]]
+; DEFAULT-NEXT: [[TMP26:%.*]] = mul i32 [[MUL_X]], [[TMP22]]
+; DEFAULT-NEXT: [[TMP29:%.*]] = mul i32 [[MUL_X]], [[TMP25]]
; DEFAULT-NEXT: [[TMP17:%.*]] = zext i32 [[TMP15]] to i64
; DEFAULT-NEXT: [[TMP18:%.*]] = zext i32 [[TMP16]] to i64
+; DEFAULT-NEXT: [[TMP23:%.*]] = zext i32 [[TMP26]] to i64
+; DEFAULT-NEXT: [[TMP24:%.*]] = zext i32 [[TMP29]] to i64
; DEFAULT-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP17]]
; DEFAULT-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP18]]
+; DEFAULT-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP23]]
+; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP24]]
; DEFAULT-NEXT: store i32 1, ptr [[TMP19]], align 4
; DEFAULT-NEXT: store i32 1, ptr [[TMP20]], align 4
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; DEFAULT-NEXT: store i32 1, ptr [[TMP27]], align 4
+; DEFAULT-NEXT: store i32 1, ptr [[TMP28]], align 4
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; DEFAULT-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
@@ -305,7 +315,7 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[ENTRY:.*]]:
; DEFAULT-NEXT: [[MUL:%.*]] = mul i32 [[X]], [[X]]
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; DEFAULT: [[VECTOR_SCEVCHECK]]:
; DEFAULT-NEXT: [[TMP1:%.*]] = mul i32 [[X]], [[X]]
@@ -326,7 +336,7 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP13:%.*]] = or i1 [[TMP9]], [[TMP12]]
; DEFAULT-NEXT: br i1 [[TMP13]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: [[TMP14:%.*]] = trunc i64 [[N_VEC]] to i32
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -334,16 +344,28 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
; DEFAULT-NEXT: [[TMP15:%.*]] = add i32 [[OFFSET_IDX]], 1
+; DEFAULT-NEXT: [[TMP26:%.*]] = add i32 [[OFFSET_IDX]], 2
+; DEFAULT-NEXT: [[TMP27:%.*]] = add i32 [[OFFSET_IDX]], 3
; DEFAULT-NEXT: [[TMP17:%.*]] = add i32 [[OFFSET_IDX]], 1
+; DEFAULT-NEXT: [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], 2
+; DEFAULT-NEXT: [[TMP31:%.*]] = add i32 [[OFFSET_IDX]], 3
; DEFAULT-NEXT: [[TMP18:%.*]] = mul i32 [[MUL]], [[OFFSET_IDX]]
; DEFAULT-NEXT: [[TMP19:%.*]] = mul i32 [[MUL]], [[TMP17]]
+; DEFAULT-NEXT: [[TMP34:%.*]] = mul i32 [[MUL]], [[TMP30]]
+; DEFAULT-NEXT: [[TMP25:%.*]] = mul i32 [[MUL]], [[TMP31]]
; DEFAULT-NEXT: [[TMP20:%.*]] = zext i32 [[TMP18]] to i64
; DEFAULT-NEXT: [[TMP21:%.*]] = zext i32 [[TMP19]] to i64
+; DEFAULT-NEXT: [[TMP28:%.*]] = zext i32 [[TMP34]] to i64
+; DEFAULT-NEXT: [[TMP29:%.*]] = zext i32 [[TMP25]] to i64
; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP20]]
; DEFAULT-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP21]]
+; DEFAULT-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP28]]
+; DEFAULT-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP29]]
; DEFAULT-NEXT: store i32 [[OFFSET_IDX]], ptr [[TMP22]], align 4
; DEFAULT-NEXT: store i32 [[TMP15]], ptr [[TMP23]], align 4
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; DEFAULT-NEXT: store i32 [[TMP26]], ptr [[TMP32]], align 4
+; DEFAULT-NEXT: store i32 [[TMP27]], ptr [[TMP33]], align 4
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; DEFAULT-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
@@ -415,7 +437,7 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[ENTRY:.*]]:
; DEFAULT-NEXT: [[ADD:%.*]] = add i32 [[X]], 1
; DEFAULT-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; DEFAULT: [[VECTOR_SCEVCHECK]]:
; DEFAULT-NEXT: [[TMP1:%.*]] = sub i32 -1, [[X]]
@@ -435,7 +457,7 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
; DEFAULT-NEXT: br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; DEFAULT-NEXT: [[TMP13:%.*]] = trunc i64 [[N_VEC]] to i32
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -443,16 +465,28 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
; DEFAULT-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], 1
+; DEFAULT-NEXT: [[TMP25:%.*]] = add i32 [[OFFSET_IDX]], 2
+; DEFAULT-NEXT: [[TMP26:%.*]] = add i32 [[OFFSET_IDX]], 3
; DEFAULT-NEXT: [[TMP16:%.*]] = add i32 [[OFFSET_IDX]], 1
+; DEFAULT-NEXT: [[TMP29:%.*]] = add i32 [[OFFSET_IDX]], 2
+; DEFAULT-NEXT: [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], 3
; DEFAULT-NEXT: [[TMP17:%.*]] = mul i32 [[ADD]], [[OFFSET_IDX]]
; DEFAULT-NEXT: [[TMP18:%.*]] = mul i32 [[ADD]], [[TMP16]]
+; DEFAULT-NEXT: [[TMP33:%.*]] = mul i32 [[ADD]], [[TMP29]]
+; DEFAULT-NEXT: [[TMP24:%.*]] = mul i32 [[ADD]], [[TMP30]]
; DEFAULT-NEXT: [[TMP19:%.*]] = zext i32 [[TMP17]] to i64
; DEFAULT-NEXT: [[TMP20:%.*]] = zext i32 [[TMP18]] to i64
+; DEFAULT-NEXT: [[TMP27:%.*]] = zext i32 [[TMP33]] to i64
+; DEFAULT-NEXT: [[TMP28:%.*]] = zext i32 [[TMP24]] to i64
; DEFAULT-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP19]]
; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP20]]
+; DEFAULT-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP27]]
+; DEFAULT-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP28]]
; DEFAULT-NEXT: store i32 [[OFFSET_IDX]], ptr [[TMP21]], align 4
; DEFAULT-NEXT: store i32 [[TMP14]], ptr [[TMP22]], align 4
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; DEFAULT-NEXT: store i32 [[TMP25]], ptr [[TMP31]], align 4
+; DEFAULT-NEXT: store i32 [[TMP26]], ptr [[TMP32]], align 4
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; DEFAULT-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
@@ -524,7 +558,7 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
; DEFAULT-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; DEFAULT-NEXT: [[ENTRY:.*]]:
; DEFAULT-NEXT: [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 2
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 4
; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; DEFAULT: [[VECTOR_SCEVCHECK]]:
; DEFAULT-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
@@ -536,18 +570,24 @@ define void @exit_cond_zext_iv(ptr %dst, i64 %N) {
; DEFAULT-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
; DEFAULT-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; DEFAULT: [[VECTOR_PH]]:
-; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 2
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 4
; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX1]], [[N_MOD_VF]]
; DEFAULT-NEXT: [[TMP6:%.*]] = trunc i64 [[N_VEC]] to i32
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
; DEFAULT: [[VECTOR_BODY]]:
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; DEFAULT-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; DEFAULT-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 2
+; DEFAULT-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 3
; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[INDEX]], i32 2
; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP7]], i32 2
+; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP11]], i32 2
+; DEFAULT-NEXT: [[TMP13:%.*]] = getelementptr { [100 x i32], i32, i32 }, ptr [[DST]], i64 [[TMP14]], i32 2
; DEFAULT-NEXT: store i32 0, ptr [[TMP8]], align 8
; DEFAULT-NEXT: store i32 0, ptr [[TMP9]], align 8
-; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; DEFAULT-NEXT: store i32 0, ptr [[TMP12]], align 8
+; DEFAULT-NEXT: store i32 0, ptr [[TMP13]], align 8
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; DEFAULT-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; DEFAULT-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; DEFAULT: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 9e8a95b6b9a47..54842aa9f5f8f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -353,7 +353,7 @@ define void @zext_iv_increment(ptr %dst, i64 %N) {
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; CHECK: [[VECTOR_SCEVCHECK]]:
; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
@@ -365,18 +365,24 @@ define void @zext_iv_increment(ptr %dst, i64 %N) {
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 2
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX1]], [[N_MOD_VF]]
; CHECK-NEXT: [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr { i32, i32, i32 }, ptr [[DST]], i64 [[INDEX]], i32 2
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr { i32, i32, i32 }, ptr [[DST]], i64 [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr { i32, i32, i32 }, ptr [[DST]], i64 [[TMP14]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr { i32, i32, i32 }, ptr [[DST]], i64 [[TMP15]], i32 2
; CHECK-NEXT: store i32 0, ptr [[TMP9]], align 8
; CHECK-NEXT: store i32 0, ptr [[TMP10]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: store i32 0, ptr [[TMP12]], align 8
+; CHECK-NEXT: store i32 0, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index 840c4b41713ce..43117f295f28b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -1694,10 +1694,10 @@ define void @not_dotp_not_phi2(ptr %matrix, i32 %n) #0 {
; CHECK-INTERLEAVED-NEXT: [[A_EXT:%.*]] = sext i8 [[LOAD_A]] to i32
; CHECK-INTERLEAVED-NEXT: [[A_EXT1:%.*]] = sext i8 [[LOAD_A1]] to i32
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64
-; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK-INTERLEAVED: vector.ph:
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; CHECK-INTERLEAVED-NEXT: [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl i64 [[N_VEC]], 4
@@ -1707,35 +1707,63 @@ define void @not_dotp_not_phi2(ptr %matrix, i32 %n) #0 {
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI4:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 16
+; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = add i64 [[OFFSET_IDX]], 32
+; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = add i64 [[OFFSET_IDX]], 48
; CHECK-INTERLEAVED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 [[OFFSET_IDX]]
; CHECK-INTERLEAVED-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 [[TMP25]]
+; CHECK-INTERLEAVED-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[MATRIX]], i64 [[TMP28]]
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 1
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 1
+; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 1
+; CHECK-INTERLEAVED-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 1
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 2
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 2
+; CHECK-INTERLEAVED-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 2
+; CHECK-INTERLEAVED-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 2
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP4]], align 1
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = load i8, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP37:%.*]] = load i8, ptr [[TMP29]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP40:%.*]] = load i8, ptr [[TMP32]], align 1
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = sext i8 [[TMP8]] to i32
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = sext i8 [[TMP9]] to i32
+; CHECK-INTERLEAVED-NEXT: [[TMP41:%.*]] = sext i8 [[TMP37]] to i32
+; CHECK-INTERLEAVED-NEXT: [[TMP44:%.*]] = sext i8 [[TMP40]] to i32
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = mul nsw i32 [[A_EXT]], [[TMP10]]
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = mul nsw i32 [[A_EXT]], [[TMP11]]
+; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = mul nsw i32 [[A_EXT]], [[TMP41]]
+; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = mul nsw i32 [[A_EXT]], [[TMP44]]
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = add i32 [[TMP12]], [[VEC_PHI]]
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = add i32 [[TMP13]], [[VEC_PHI2]]
+; CHECK-INTERLEAVED-NEXT: [[TMP30:%.*]] = add i32 [[TMP26]], [[VEC_PHI4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = add i32 [[TMP27]], [[VEC_PHI3]]
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP6]], align 1
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP34:%.*]] = load i8, ptr [[TMP33]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP36]], align 1
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = sext i8 [[TMP16]] to i32
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = sext i8 [[TMP17]] to i32
+; CHECK-INTERLEAVED-NEXT: [[TMP38:%.*]] = sext i8 [[TMP34]] to i32
+; CHECK-INTERLEAVED-NEXT: [[TMP39:%.*]] = sext i8 [[TMP35]] to i32
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = mul nsw i32 [[A_EXT1]], [[TMP18]]
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = mul nsw i32 [[A_EXT1]], [[TMP19]]
+; CHECK-INTERLEAVED-NEXT: [[TMP42:%.*]] = mul nsw i32 [[A_EXT1]], [[TMP38]]
+; CHECK-INTERLEAVED-NEXT: [[TMP43:%.*]] = mul nsw i32 [[A_EXT1]], [[TMP39]]
; CHECK-INTERLEAVED-NEXT: [[TMP22]] = add i32 [[TMP20]], [[TMP14]]
; CHECK-INTERLEAVED-NEXT: [[TMP23]] = add i32 [[TMP21]], [[TMP15]]
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-INTERLEAVED-NEXT: [[TMP46]] = add i32 [[TMP42]], [[TMP30]]
+; CHECK-INTERLEAVED-NEXT: [[TMP47]] = add i32 [[TMP43]], [[TMP31]]
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK-INTERLEAVED: middle.block:
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP23]], [[TMP22]]
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX7:%.*]] = add i32 [[TMP46]], [[BIN_RDX]]
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX8:%.*]] = add i32 [[TMP47]], [[BIN_RDX7]]
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-INTERLEAVED: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
index 9a60827951036..2d330967cd673 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/replicating-load-store-costs.ll
@@ -13,17 +13,29 @@ define void @replicating_load_used_as_store_addr(ptr noalias %A) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP11]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP12]], 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr ptr, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr ptr, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr ptr, ptr [[A]], i64 [[TMP12]]
; CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP3]], align 8
; CHECK-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP19]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = load ptr, ptr [[TMP10]], align 8
; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP15]] to i32
+; CHECK-NEXT: [[TMP18:%.*]] = trunc i64 [[TMP16]] to i32
; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP5]], align 4
; CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP6]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP13]], align 4
+; CHECK-NEXT: store i32 [[TMP18]], ptr [[TMP14]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -62,7 +74,7 @@ define void @replicating_load_used_as_store_addr_2(ptr noalias %invar.dst, ptr n
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 123
; CHECK-NEXT: store i32 [[TMP4]], ptr [[INVAR_DST]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -110,7 +122,7 @@ define void @replicating_load_used_as_store_addr_3(ptr noalias %src, ptr noalias
; CHECK-NEXT: store i8 0, ptr [[TMP7]], align 1
; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP5]] to i8
; CHECK-NEXT: store i8 [[TMP8]], ptr [[INVAR_DST]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-illegal-type.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-illegal-type.ll
index e735fad4e56ae..61cbdff3e0679 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-illegal-type.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-illegal-type.ll
@@ -9,10 +9,16 @@ define void @loop_sve_i128(ptr nocapture %ptr, i64 %N) {
; CHECK: vector.body
; CHECK: %[[LOAD1:.*]] = load i128, ptr {{.*}}
; CHECK-NEXT: %[[LOAD2:.*]] = load i128, ptr {{.*}}
+; CHECK-NEXT: %[[LOAD3:.*]] = load i128, ptr {{.*}}
+; CHECK-NEXT: %[[LOAD4:.*]] = load i128, ptr {{.*}}
; CHECK-NEXT: %[[ADD1:.*]] = add nsw i128 %[[LOAD1]], 42
; CHECK-NEXT: %[[ADD2:.*]] = add nsw i128 %[[LOAD2]], 42
+; CHECK-NEXT: %[[ADD3:.*]] = add nsw i128 %[[LOAD3]], 42
+; CHECK-NEXT: %[[ADD4:.*]] = add nsw i128 %[[LOAD4]], 42
; CHECK-NEXT: store i128 %[[ADD1]], ptr {{.*}}
; CHECK-NEXT: store i128 %[[ADD2]], ptr {{.*}}
+; CHECK-NEXT: store i128 %[[ADD3]], ptr {{.*}}
+; CHECK-NEXT: store i128 %[[ADD4]], ptr {{.*}}
entry:
br label %for.body
@@ -36,10 +42,16 @@ define void @loop_sve_f128(ptr nocapture %ptr, i64 %N) {
; CHECK: vector.body
; CHECK: %[[LOAD1:.*]] = load fp128, ptr
; CHECK-NEXT: %[[LOAD2:.*]] = load fp128, ptr
+; CHECK-NEXT: %[[LOAD3:.*]] = load fp128, ptr
+; CHECK-NEXT: %[[LOAD4:.*]] = load fp128, ptr
; CHECK-NEXT: %[[FSUB1:.*]] = fsub fp128 %[[LOAD1]], -0.000000e+00
; CHECK-NEXT: %[[FSUB2:.*]] = fsub fp128 %[[LOAD2]], -0.000000e+00
+; CHECK-NEXT: %[[FSUB3:.*]] = fsub fp128 %[[LOAD3]], -0.000000e+00
+; CHECK-NEXT: %[[FSUB4:.*]] = fsub fp128 %[[LOAD4]], -0.000000e+00
; CHECK-NEXT: store fp128 %[[FSUB1]], ptr {{.*}}
; CHECK-NEXT: store fp128 %[[FSUB2]], ptr {{.*}}
+; CHECK-NEXT: store fp128 %[[FSUB3]], ptr {{.*}}
+; CHECK-NEXT: store fp128 %[[FSUB4]], ptr {{.*}}
entry:
br label %for.body
@@ -63,8 +75,12 @@ define void @loop_invariant_sve_i128(ptr nocapture %ptr, i128 %val, i64 %N) {
; CHECK: vector.body
; CHECK: %[[GEP1:.*]] = getelementptr inbounds i128, ptr %ptr
; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds i128, ptr %ptr
+; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds i128, ptr %ptr
+; CHECK-NEXT: %[[GEP4:.*]] = getelementptr inbounds i128, ptr %ptr
; CHECK-NEXT: store i128 %val, ptr %[[GEP1]]
; CHECK-NEXT: store i128 %val, ptr %[[GEP2]]
+; CHECK-NEXT: store i128 %val, ptr %[[GEP3]]
+; CHECK-NEXT: store i128 %val, ptr %[[GEP4]]
entry:
br label %for.body
More information about the llvm-commits
mailing list