[llvm] [VPlan] Cost truncated widened inductions via ::computeCost. (PR #212786)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 02:59:55 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/212786
>From 9ae038f686efeb21a66c08a168077c7e380a753b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 7 Jun 2026 17:13:52 +0200
Subject: [PATCH 1/3] [VPlan] Cost all widened inductions via ::computeCost.
Also compute costs for truncated inductions in ::computeCost, removing
the fallback to the legacy cost model unless the loop is fully unrolled.
Note that this changes vectorization decisions in both direction, e.g.
* now vectorized `@tail_predicate_without_optsize`: legacy costs <16 x
i64>, whereas we generate a narrow IV <16 x i8>, which is much
cheaper
* no longer vectorized `@second_lshr_operand_zero_via_scev()`: we
generate 2 IVs (one truncated and one not truncated), which is more
expensive than the single IV LV assumes (note that previously we
would ignore the cost of the trunc in the cost computation, because
the trunc was replaced by the wide truncated IV, which was assumed
free.
Both cases are due to more accurate cost computations.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 22 +--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 -
.../LoopVectorize/AArch64/induction-costs.ll | 16 +-
.../LoopVectorize/ARM/optsize_minsize.ll | 141 ++++++++++++++++--
.../LoopVectorize/X86/conversion-cost.ll | 21 ++-
.../X86/cost-constant-known-via-scev.ll | 48 ++----
.../LoopVectorize/X86/induction-costs.ll | 36 ++---
.../LoopVectorize/induction-cost.ll | 14 +-
8 files changed, 198 insertions(+), 103 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a0abc151ace19..5ecd842341731 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5662,6 +5662,9 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
// and skip all recipes that represent induction phis and increments (the
// former case) later on, if they exist, to avoid counting them twice.
// Similarly we pre-compute the cost of any optimized truncates.
+ // Inductions that are represented by a VPWidenIntOrFpInductionRecipe are an
+ // exception: their cost is computed by the recipe's computeCost (see below),
+ // so they are not precomputed here.
// TODO: Switch to more accurate costing based on VPlan.
// If the vector loop gets executed exactly once with the given VF, ignore the
@@ -5672,26 +5675,23 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
bool IsFullyUnrolled = TC == VF && !Plan.hasTailFolded();
SmallPtrSet<const Value *, 4> WidenedIVs;
- bool HasTruncatedIV = false;
if (IsFullyUnrolled) {
addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
CostCtx.SkipCostComputation);
} else {
// Inductions represented by a VPWidenIntOrFpInductionRecipe have their cost
// computed by the recipe, so collect their phis to skip the legacy
- // increment cost below. If any induction is truncated the VPlan-based cost
- // will diverge. Still fall back to the legacy cost model for now.
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- for (VPRecipeBase &R : *LoopRegion->getEntryBasicBlock())
- if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
- HasTruncatedIV |= WideIV->getTruncInst() != nullptr;
- if (PHINode *IVPhi = WideIV->getPHINode())
- WidenedIVs.insert(IVPhi);
- }
+ // increment cost below.
+ if (VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion())
+ for (VPRecipeBase &R : *LoopRegion->getEntryBasicBlock())
+ if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
+ if (PHINode *IVPhi = WideIV->getPHINode())
+ WidenedIVs.insert(IVPhi);
+ }
}
for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
- if (!HasTruncatedIV && WidenedIVs.contains(IV))
+ if (!IsFullyUnrolled && WidenedIVs.contains(IV))
continue;
Instruction *IVInc = cast<Instruction>(
IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5d692ea89ac2e..ecf18790f21ff 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2965,9 +2965,6 @@ VPWidenIntOrFpInductionRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
// A widened induction generates a vector phi and increments it by the
// splatted step each iteration.
- // TODO: Also handle truncated inductions.
- assert(!getTruncInst() &&
- "truncated inductions should be costed by the legacy model");
const InductionDescriptor &ID = getInductionDescriptor();
InstructionCost Cost = Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
Type *StepTy = getScalarType();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index 29f54d4103db4..9304fa7d9934a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -212,14 +212,14 @@ define void @wide_truncated_iv(ptr %dst) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add <8 x i8> [[VEC_IND]], splat (i8 8)
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add <16 x i8> [[VEC_IND]], splat (i8 16)
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 8
-; CHECK-NEXT: store <8 x i8> [[VEC_IND]], ptr [[TMP0]], align 1
-; CHECK-NEXT: store <8 x i8> [[STEP_ADD]], ptr [[TMP1]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i8> [[STEP_ADD]], splat (i8 8)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT: store <16 x i8> [[VEC_IND]], ptr [[TMP0]], align 1
+; CHECK-NEXT: store <16 x i8> [[STEP_ADD]], ptr [[TMP1]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <16 x i8> [[STEP_ADD]], splat (i8 16)
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 192
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
@@ -884,7 +884,7 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8]]
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF17:%.*]] = urem i64 [[TMP0]], 8
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
index 83be0708774f1..2526acdb47079 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
@@ -181,24 +181,137 @@ for.cond.cleanup:
define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n) {
; DEFAULT-LABEL: define void @tail_predicate_without_optsize(
; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) {
-; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[ENTRY:.*:]]
; DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
; DEFAULT: [[FOR_BODY]]:
-; DEFAULT-NEXT: [[TMP69:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; DEFAULT-NEXT: [[TMP0:%.*]] = trunc nuw nsw i64 [[TMP69]] to i8
-; DEFAULT-NEXT: [[MUL:%.*]] = mul i8 [[A]], [[TMP0]]
-; DEFAULT-NEXT: [[SHR:%.*]] = lshr i8 [[TMP0]], 1
-; DEFAULT-NEXT: [[MUL5:%.*]] = mul i8 [[SHR]], [[B]]
-; DEFAULT-NEXT: [[ADD:%.*]] = add i8 [[MUL5]], [[MUL]]
-; DEFAULT-NEXT: [[SHR7:%.*]] = lshr i8 [[TMP0]], 2
-; DEFAULT-NEXT: [[MUL9:%.*]] = mul i8 [[SHR7]], [[C]]
-; DEFAULT-NEXT: [[TMP71:%.*]] = add i8 [[ADD]], [[MUL9]]
-; DEFAULT-NEXT: [[TMP70:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP69]]
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i8> poison, i8 [[B]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT1]], <16 x i8> poison, <16 x i32> zeroinitializer
+; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <16 x i8> poison, i8 [[C]], i64 0
+; DEFAULT-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT3]], <16 x i8> poison, <16 x i32> zeroinitializer
+; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; DEFAULT: [[VECTOR_BODY]]:
+; DEFAULT-NEXT: [[TMP0:%.*]] = mul <16 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>
+; DEFAULT-NEXT: [[TMP1:%.*]] = mul <16 x i8> <i8 0, i8 0, i8 1, i8 1, i8 2, i8 2, i8 3, i8 3, i8 4, i8 4, i8 5, i8 5, i8 6, i8 6, i8 7, i8 7>, [[BROADCAST_SPLAT2]]
+; DEFAULT-NEXT: [[TMP2:%.*]] = add <16 x i8> [[TMP1]], [[TMP0]]
+; DEFAULT-NEXT: [[TMP3:%.*]] = mul <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 2, i8 2, i8 2, i8 2, i8 3, i8 3, i8 3, i8 3>, [[BROADCAST_SPLAT4]]
+; DEFAULT-NEXT: [[TMP4:%.*]] = add <16 x i8> [[TMP2]], [[TMP3]]
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; DEFAULT: [[PRED_STORE_IF]]:
+; DEFAULT-NEXT: [[TMP5:%.*]] = extractelement <16 x i8> [[TMP4]], i64 0
+; DEFAULT-NEXT: store i8 [[TMP5]], ptr [[P]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; DEFAULT: [[PRED_STORE_CONTINUE]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT: [[PRED_STORE_IF5]]:
+; DEFAULT-NEXT: [[TMP70:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; DEFAULT-NEXT: [[TMP71:%.*]] = extractelement <16 x i8> [[TMP4]], i64 1
; DEFAULT-NEXT: store i8 [[TMP71]], ptr [[TMP70]], align 1
-; DEFAULT-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[TMP69]], 1
-; DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 15
-; DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY]]
+; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP]]
; DEFAULT: [[FOR_COND_CLEANUP]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; DEFAULT: [[PRED_STORE_IF7]]:
+; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; DEFAULT-NEXT: [[TMP9:%.*]] = extractelement <16 x i8> [[TMP4]], i64 2
+; DEFAULT-NEXT: store i8 [[TMP9]], ptr [[TMP8]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; DEFAULT: [[PRED_STORE_CONTINUE8]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; DEFAULT: [[PRED_STORE_IF9]]:
+; DEFAULT-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; DEFAULT-NEXT: [[TMP11:%.*]] = extractelement <16 x i8> [[TMP4]], i64 3
+; DEFAULT-NEXT: store i8 [[TMP11]], ptr [[TMP10]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; DEFAULT: [[PRED_STORE_CONTINUE10]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; DEFAULT: [[PRED_STORE_IF11]]:
+; DEFAULT-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; DEFAULT-NEXT: [[TMP13:%.*]] = extractelement <16 x i8> [[TMP4]], i64 4
+; DEFAULT-NEXT: store i8 [[TMP13]], ptr [[TMP12]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; DEFAULT: [[PRED_STORE_CONTINUE12]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; DEFAULT: [[PRED_STORE_IF13]]:
+; DEFAULT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; DEFAULT-NEXT: [[TMP15:%.*]] = extractelement <16 x i8> [[TMP4]], i64 5
+; DEFAULT-NEXT: store i8 [[TMP15]], ptr [[TMP14]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE14]]
+; DEFAULT: [[PRED_STORE_CONTINUE14]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
+; DEFAULT: [[PRED_STORE_IF15]]:
+; DEFAULT-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; DEFAULT-NEXT: [[TMP17:%.*]] = extractelement <16 x i8> [[TMP4]], i64 6
+; DEFAULT-NEXT: store i8 [[TMP17]], ptr [[TMP16]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; DEFAULT: [[PRED_STORE_CONTINUE16]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
+; DEFAULT: [[PRED_STORE_IF17]]:
+; DEFAULT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
+; DEFAULT-NEXT: [[TMP19:%.*]] = extractelement <16 x i8> [[TMP4]], i64 7
+; DEFAULT-NEXT: store i8 [[TMP19]], ptr [[TMP18]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE18]]
+; DEFAULT: [[PRED_STORE_CONTINUE18]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
+; DEFAULT: [[PRED_STORE_IF19]]:
+; DEFAULT-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 8
+; DEFAULT-NEXT: [[TMP21:%.*]] = extractelement <16 x i8> [[TMP4]], i64 8
+; DEFAULT-NEXT: store i8 [[TMP21]], ptr [[TMP20]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE20]]
+; DEFAULT: [[PRED_STORE_CONTINUE20]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]
+; DEFAULT: [[PRED_STORE_IF21]]:
+; DEFAULT-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 9
+; DEFAULT-NEXT: [[TMP23:%.*]] = extractelement <16 x i8> [[TMP4]], i64 9
+; DEFAULT-NEXT: store i8 [[TMP23]], ptr [[TMP22]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE22]]
+; DEFAULT: [[PRED_STORE_CONTINUE22]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]
+; DEFAULT: [[PRED_STORE_IF23]]:
+; DEFAULT-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 10
+; DEFAULT-NEXT: [[TMP25:%.*]] = extractelement <16 x i8> [[TMP4]], i64 10
+; DEFAULT-NEXT: store i8 [[TMP25]], ptr [[TMP24]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE24]]
+; DEFAULT: [[PRED_STORE_CONTINUE24]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]
+; DEFAULT: [[PRED_STORE_IF25]]:
+; DEFAULT-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 11
+; DEFAULT-NEXT: [[TMP27:%.*]] = extractelement <16 x i8> [[TMP4]], i64 11
+; DEFAULT-NEXT: store i8 [[TMP27]], ptr [[TMP26]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE26]]
+; DEFAULT: [[PRED_STORE_CONTINUE26]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]
+; DEFAULT: [[PRED_STORE_IF27]]:
+; DEFAULT-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 12
+; DEFAULT-NEXT: [[TMP29:%.*]] = extractelement <16 x i8> [[TMP4]], i64 12
+; DEFAULT-NEXT: store i8 [[TMP29]], ptr [[TMP28]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE28]]
+; DEFAULT: [[PRED_STORE_CONTINUE28]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]
+; DEFAULT: [[PRED_STORE_IF29]]:
+; DEFAULT-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 13
+; DEFAULT-NEXT: [[TMP31:%.*]] = extractelement <16 x i8> [[TMP4]], i64 13
+; DEFAULT-NEXT: store i8 [[TMP31]], ptr [[TMP30]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE30]]
+; DEFAULT: [[PRED_STORE_CONTINUE30]]:
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF31:.*]], label %[[PRED_STORE_CONTINUE32:.*]]
+; DEFAULT: [[PRED_STORE_IF31]]:
+; DEFAULT-NEXT: [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 14
+; DEFAULT-NEXT: [[TMP33:%.*]] = extractelement <16 x i8> [[TMP4]], i64 14
+; DEFAULT-NEXT: store i8 [[TMP33]], ptr [[TMP32]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE32]]
+; DEFAULT: [[PRED_STORE_CONTINUE32]]:
+; DEFAULT-NEXT: br i1 false, label %[[PRED_STORE_IF33:.*]], label %[[PRED_STORE_CONTINUE34:.*]]
+; DEFAULT: [[PRED_STORE_IF33]]:
+; DEFAULT-NEXT: [[TMP34:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 15
+; DEFAULT-NEXT: [[TMP35:%.*]] = extractelement <16 x i8> [[TMP4]], i64 15
+; DEFAULT-NEXT: store i8 [[TMP35]], ptr [[TMP34]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE34]]
+; DEFAULT: [[PRED_STORE_CONTINUE34]]:
+; DEFAULT-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; DEFAULT: [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP1:.*]]
+; DEFAULT: [[FOR_COND_CLEANUP1]]:
; DEFAULT-NEXT: ret void
;
; OPTSIZE-LABEL: define void @tail_predicate_without_optsize(
diff --git a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
index a8e32ca9a0c8c..1a4d64c32dcaf 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
@@ -15,21 +15,30 @@ define void @conversion_cost1(i32 %n, ptr nocapture %A, ptr nocapture %B) {
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], 32
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], 64
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 32
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 64
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP4:%.*]] = add i64 3, [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i8> [ <i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <16 x i8> [ <i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add <16 x i8> [[VEC_IND]], splat (i8 16)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add <16 x i8> [[STEP_ADD]], splat (i8 16)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add <16 x i8> [[STEP_ADD_2]], splat (i8 16)
; CHECK-NEXT: [[TMP5:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP5]]
-; CHECK-NEXT: store <32 x i8> [[VEC_IND]], ptr [[TMP6]], align 1
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i8> [[VEC_IND]], splat (i8 32)
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 32
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 48
+; CHECK-NEXT: store <16 x i8> [[VEC_IND]], ptr [[TMP6]], align 1
+; CHECK-NEXT: store <16 x i8> [[STEP_ADD]], ptr [[TMP15]], align 1
+; CHECK-NEXT: store <16 x i8> [[STEP_ADD_2]], ptr [[TMP16]], align 1
+; CHECK-NEXT: store <16 x i8> [[STEP_ADD_3]], ptr [[TMP17]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <16 x i8> [[STEP_ADD_3]], splat (i8 16)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
index dbf1eaea669b7..7c7fb11d1a8f3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
@@ -60,41 +60,24 @@ exit:
; Test case for https://github.com/llvm/llvm-project/issues/109528.
define i64 @second_lshr_operand_zero_via_scev() {
; CHECK-LABEL: define i64 @second_lshr_operand_zero_via_scev() {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[EXT_0:%.*]] = sext i8 0 to i32
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[STEP_ADD4:%.*]] = add <2 x i32> [[VEC_IND2]], splat (i32 2)
-; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <2 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i64> [[STEP_ADD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i64> [[VEC_IND]], splat (i64 3)
-; CHECK-NEXT: [[TMP3:%.*]] = and <2 x i64> [[STEP_ADD]], splat (i64 3)
-; CHECK-NEXT: [[TMP4:%.*]] = lshr <2 x i32> [[VEC_IND2]], zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = lshr <2 x i32> [[STEP_ADD4]], zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = zext <2 x i32> [[TMP4]] to <2 x i64>
-; CHECK-NEXT: [[TMP7:%.*]] = zext <2 x i32> [[TMP5]] to <2 x i64>
-; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP0]], <2 x i64> [[TMP2]], <2 x i64> [[TMP6]]
-; CHECK-NEXT: [[TMP9:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[TMP3]], <2 x i64> [[TMP7]]
-; CHECK-NEXT: [[TMP10]] = or <2 x i64> [[TMP8]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP11]] = or <2 x i64> [[TMP9]], [[VEC_PHI1]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], splat (i64 2)
-; CHECK-NEXT: [[VEC_IND_NEXT3]] = add <2 x i32> [[STEP_ADD4]], splat (i32 2)
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT: [[AND:%.*]] = and i64 [[IV]], 3
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[TMP0]], [[EXT_0]]
+; CHECK-NEXT: [[CONV_1:%.*]] = zext i32 [[SHR]] to i64
+; CHECK-NEXT: [[RED_NEXT_V:%.*]] = select i1 [[C]], i64 [[AND]], i64 [[CONV_1]]
+; CHECK-NEXT: [[RED_NEXT]] = or i64 [[RED_NEXT_V]], [[RED]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[BIN_RDX:%.*]] = or <2 x i64> [[TMP11]], [[TMP10]]
-; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[BIN_RDX]])
-; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP13:%.*]] = phi i64 [ [[RED_NEXT]], %[[LOOP]] ]
; CHECK-NEXT: ret i64 [[TMP13]]
;
entry:
@@ -125,5 +108,4 @@ exit:
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 1a5188687b508..cb9041ea888ca 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -8,7 +8,7 @@ define i32 @iv_used_widened_and_truncated(ptr %dst, i64 %N) #0 {
; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
-; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 4
+; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 8
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[ENTRY:.*]]
; CHECK: [[ENTRY]]:
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32
@@ -44,29 +44,29 @@ define i32 @iv_used_widened_and_truncated(ptr %dst, i64 %N) #0 {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[N_MOD_VF8:%.*]] = urem i64 [[TMP0]], 4
+; CHECK-NEXT: [[N_MOD_VF8:%.*]] = urem i64 [[TMP0]], 8
; CHECK-NEXT: [[N_VEC9:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF8]]
-; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
-; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[DOTSPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
-; CHECK-NEXT: [[DOTSPLATINSERT14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
-; CHECK-NEXT: [[DOTSPLAT15:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT14]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION16:%.*]] = add <4 x i32> [[DOTSPLAT15]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <8 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT12]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION14:%.*]] = add <8 x i32> [[BROADCAST_SPLAT13]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[INDEX11:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND12:%.*]] = phi <4 x i64> [ [[INDUCTION]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT13:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND17:%.*]] = phi <4 x i32> [ [[INDUCTION16]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT18:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr { i32, [8 x i32] }, ptr [[DST]], <4 x i64> [[VEC_IND12]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[VEC_IND17]], <4 x ptr> align 8 [[TMP7]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT19]] = add nuw i64 [[INDEX11]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT13]] = add <4 x i64> [[VEC_IND12]], splat (i64 4)
-; CHECK-NEXT: [[VEC_IND_NEXT18]] = add <4 x i32> [[VEC_IND17]], splat (i32 4)
+; CHECK-NEXT: [[INDEX15:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND16:%.*]] = phi <8 x i64> [ [[INDUCTION]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT20:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND17:%.*]] = phi <8 x i32> [ [[INDUCTION14]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT21:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[WIDE_GEP18:%.*]] = getelementptr { i32, [8 x i32] }, ptr [[DST]], <8 x i64> [[VEC_IND16]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[VEC_IND17]], <8 x ptr> align 8 [[WIDE_GEP18]], <8 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT19]] = add nuw i64 [[INDEX15]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT20]] = add <8 x i64> [[VEC_IND16]], splat (i64 8)
+; CHECK-NEXT: [[VEC_IND_NEXT21]] = add <8 x i32> [[VEC_IND17]], splat (i32 8)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT19]], [[N_VEC9]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -495,7 +495,7 @@ define i32 @test_scalar_predicated_cost(i64 %x, i64 %y, ptr %A) #0 {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br i1 false, label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; CHECK-NEXT: br i1 false, label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i64> poison, i64 [[Y]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/induction-cost.ll b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
index 2e52b8878525f..8fbe1ba716cd8 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost.*WIDEN-INDUCTION)|(induction instruction)" --filter-out-after "LV: Selecting VF"
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost.*WIDEN-INDUCTION)|(induction instruction)" --filter-out-after "LV: Using user VF"
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s
@@ -46,10 +46,7 @@ exit:
define void @trunc_induction(ptr noalias %dst, i64 %n) {
; CHECK-LABEL: 'trunc_induction'
-; CHECK: Cost of 1 for VF 4: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 1 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 4: induction instruction %iv.trunc = trunc i64 %iv to i32
-; CHECK: Cost of 0 for VF 4: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> (truncated to i32)
+; CHECK: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]> (truncated to i32)
;
entry:
br label %loop
@@ -69,11 +66,8 @@ exit:
define void @trunc_and_wide_induction(ptr noalias %dst, ptr noalias %dst2, i64 %n) {
; CHECK-LABEL: 'trunc_and_wide_induction'
-; CHECK: Cost of 1 for VF 4: induction instruction %iv.next = add nuw nsw i64 %iv, 1
-; CHECK: Cost of 1 for VF 4: induction instruction %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK: Cost of 1 for VF 4: induction instruction %iv.trunc = trunc i64 %iv to i32
-; CHECK: Cost of 0 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK: Cost of 0 for VF 4: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32)
+; CHECK: Cost of 2 for VF 4: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
+; CHECK: Cost of 2 for VF 4: ir<%iv>.1 = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]> (truncated to i32)
;
entry:
br label %loop
>From 517bd8a1a3773bef307bd080154f68b0effe8069 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 30 Jul 2026 10:10:03 +0100
Subject: [PATCH 2/3] !fixup simplify
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5ecd842341731..2ecb3291b8abb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5673,25 +5673,24 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
// TODO: Remove this code after stepping away from the legacy cost model and
// adding code to simplify VPlans before calculating their costs.
auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
- bool IsFullyUnrolled = TC == VF && !Plan.hasTailFolded();
SmallPtrSet<const Value *, 4> WidenedIVs;
- if (IsFullyUnrolled) {
+ if (TC == VF && !Plan.hasTailFolded()) {
addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
CostCtx.SkipCostComputation);
} else {
// Inductions represented by a VPWidenIntOrFpInductionRecipe have their cost
// computed by the recipe, so collect their phis to skip the legacy
// increment cost below.
- if (VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion())
- for (VPRecipeBase &R : *LoopRegion->getEntryBasicBlock())
- if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
- if (PHINode *IVPhi = WideIV->getPHINode())
- WidenedIVs.insert(IVPhi);
- }
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ for (VPRecipeBase &R : *LoopRegion->getEntryBasicBlock())
+ if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
+ if (PHINode *IVPhi = WideIV->getPHINode())
+ WidenedIVs.insert(IVPhi);
+ }
}
for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
- if (!IsFullyUnrolled && WidenedIVs.contains(IV))
+ if (WidenedIVs.contains(IV))
continue;
Instruction *IVInc = cast<Instruction>(
IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
>From ee93d42f797a88fb174e78f79b634587e8f01f97 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 10 Aug 2026 11:37:41 +0200
Subject: [PATCH 3/3] !fixup update tests after merge
---
.../LoopVectorize/AArch64/induction-costs.ll | 2 +-
.../LoopVectorize/AArch64/optsize_minsize.ll | 48 +++++++++----------
.../LoopVectorize/ARM/optsize_minsize.ll | 20 ++++----
.../LoopVectorize/X86/conversion-cost.ll | 2 +-
.../X86/cost-constant-known-via-scev.ll | 18 +++----
.../LoopVectorize/X86/induction-costs.ll | 26 +++++-----
.../LoopVectorize/induction-cost.ll | 2 +-
7 files changed, 59 insertions(+), 59 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index c68711ccaf618..aef0099f875d8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -878,7 +878,7 @@ define i64 @live_out_extract_from_ptr_iv_increment(i64 %count, ptr %start, ptr n
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
-; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
+; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF27:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[N_MOD_VF17:%.*]] = and i64 [[TMP0]], 7
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index c26b013a1baf6..cb79aff9fed80 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -484,37 +484,37 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
; MINSIZE-NEXT: br label %[[VECTOR_PH:.*]]
; MINSIZE: [[VECTOR_PH]]:
; MINSIZE-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; MINSIZE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
-; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 15)
-; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[A]], i64 0
-; MINSIZE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
-; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[B]], i64 0
-; MINSIZE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
-; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[C]], i64 0
-; MINSIZE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
-; MINSIZE-NEXT: [[TMP2:%.*]] = call <vscale x 2 x i8> @llvm.stepvector.nxv2i8()
+; MINSIZE-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
+; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 15)
+; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[A]], i64 0
+; MINSIZE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[B]], i64 0
+; MINSIZE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[C]], i64 0
+; MINSIZE-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; MINSIZE-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
; MINSIZE-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP6]] to i8
-; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[TMP12]], i64 0
-; MINSIZE-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
+; MINSIZE-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP12]], i64 0
+; MINSIZE-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT5]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
; MINSIZE-NEXT: br label %[[VECTOR_BODY:.*]]
; MINSIZE: [[VECTOR_BODY]]:
; MINSIZE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MINSIZE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MINSIZE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i8> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; MINSIZE-NEXT: [[TMP4:%.*]] = mul <vscale x 2 x i8> [[BROADCAST_SPLAT]], [[VEC_IND]]
-; MINSIZE-NEXT: [[TMP11:%.*]] = lshr <vscale x 2 x i8> [[VEC_IND]], splat (i8 1)
-; MINSIZE-NEXT: [[TMP13:%.*]] = mul <vscale x 2 x i8> [[TMP11]], [[BROADCAST_SPLAT2]]
-; MINSIZE-NEXT: [[TMP7:%.*]] = add <vscale x 2 x i8> [[TMP13]], [[TMP4]]
-; MINSIZE-NEXT: [[TMP8:%.*]] = lshr <vscale x 2 x i8> [[VEC_IND]], splat (i8 2)
-; MINSIZE-NEXT: [[TMP9:%.*]] = mul <vscale x 2 x i8> [[TMP8]], [[BROADCAST_SPLAT4]]
-; MINSIZE-NEXT: [[TMP10:%.*]] = add <vscale x 2 x i8> [[TMP7]], [[TMP9]]
+; MINSIZE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MINSIZE-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i8> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; MINSIZE-NEXT: [[TMP4:%.*]] = mul <vscale x 16 x i8> [[BROADCAST_SPLAT]], [[VEC_IND]]
+; MINSIZE-NEXT: [[TMP11:%.*]] = lshr <vscale x 16 x i8> [[VEC_IND]], splat (i8 1)
+; MINSIZE-NEXT: [[TMP13:%.*]] = mul <vscale x 16 x i8> [[TMP11]], [[BROADCAST_SPLAT2]]
+; MINSIZE-NEXT: [[TMP7:%.*]] = add <vscale x 16 x i8> [[TMP13]], [[TMP4]]
+; MINSIZE-NEXT: [[TMP8:%.*]] = lshr <vscale x 16 x i8> [[VEC_IND]], splat (i8 2)
+; MINSIZE-NEXT: [[TMP9:%.*]] = mul <vscale x 16 x i8> [[TMP8]], [[BROADCAST_SPLAT4]]
+; MINSIZE-NEXT: [[TMP10:%.*]] = add <vscale x 16 x i8> [[TMP7]], [[TMP9]]
; MINSIZE-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
-; MINSIZE-NEXT: call void @llvm.masked.store.nxv2i8.p0(<vscale x 2 x i8> [[TMP10]], ptr align 1 [[TMP22]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; MINSIZE-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP10]], ptr align 1 [[TMP22]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
; MINSIZE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
-; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 15)
-; MINSIZE-NEXT: [[TMP24:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; MINSIZE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 15)
+; MINSIZE-NEXT: [[TMP24:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; MINSIZE-NEXT: [[TMP23:%.*]] = xor i1 [[TMP24]], true
-; MINSIZE-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i8> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; MINSIZE-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[BROADCAST_SPLAT6]]
; MINSIZE-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; MINSIZE: [[MIDDLE_BLOCK]]:
; MINSIZE-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
index 9caa73d74521b..e7683c6423575 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
@@ -182,8 +182,8 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
; DEFAULT-LABEL: define void @tail_predicate_without_optsize(
; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) {
; DEFAULT-NEXT: [[ENTRY:.*:]]
-; DEFAULT-NEXT: br label %[[FOR_BODY:.*]]
-; DEFAULT: [[FOR_BODY]]:
+; DEFAULT-NEXT: br label %[[VECTOR_PH:.*]]
+; DEFAULT: [[VECTOR_PH]]:
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i8> poison, i8 [[B]], i64 0
@@ -203,13 +203,13 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
; DEFAULT-NEXT: store i8 [[TMP5]], ptr [[P]], align 1
; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
; DEFAULT: [[PRED_STORE_CONTINUE]]:
-; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
; DEFAULT: [[PRED_STORE_IF5]]:
-; DEFAULT-NEXT: [[TMP70:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
-; DEFAULT-NEXT: [[TMP71:%.*]] = extractelement <16 x i8> [[TMP4]], i64 1
-; DEFAULT-NEXT: store i8 [[TMP71]], ptr [[TMP70]], align 1
-; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP]]
-; DEFAULT: [[FOR_COND_CLEANUP]]:
+; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; DEFAULT-NEXT: [[TMP7:%.*]] = extractelement <16 x i8> [[TMP4]], i64 1
+; DEFAULT-NEXT: store i8 [[TMP7]], ptr [[TMP6]], align 1
+; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; DEFAULT: [[PRED_STORE_CONTINUE6]]:
; DEFAULT-NEXT: br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
; DEFAULT: [[PRED_STORE_IF7]]:
; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
@@ -310,8 +310,8 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
; DEFAULT: [[PRED_STORE_CONTINUE34]]:
; DEFAULT-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; DEFAULT: [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP1:.*]]
-; DEFAULT: [[FOR_COND_CLEANUP1]]:
+; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT: [[FOR_COND_CLEANUP]]:
; DEFAULT-NEXT: ret void
;
; OPTSIZE-LABEL: define void @tail_predicate_without_optsize(
diff --git a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
index 65a871124f22c..b272d81a9b086 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
@@ -18,7 +18,7 @@ define void @conversion_cost1(i32 %n, ptr nocapture %A, ptr nocapture %B) {
; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], 64
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 31
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[TMP3]], 63
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP4:%.*]] = add i64 3, [[N_VEC]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
index 7c7fb11d1a8f3..9a195225331d2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
@@ -62,10 +62,10 @@ define i64 @second_lshr_operand_zero_via_scev() {
; CHECK-LABEL: define i64 @second_lshr_operand_zero_via_scev() {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[EXT_0:%.*]] = sext i8 0 to i32
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[LOOPS:.*]]
+; CHECK: [[LOOPS]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOPS]] ]
+; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_NEXT:%.*]], %[[LOOPS]] ]
; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[IV]], 0
; CHECK-NEXT: [[AND:%.*]] = and i64 [[IV]], 3
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[IV]] to i32
@@ -73,12 +73,12 @@ define i64 @second_lshr_operand_zero_via_scev() {
; CHECK-NEXT: [[CONV_1:%.*]] = zext i32 [[SHR]] to i64
; CHECK-NEXT: [[RED_NEXT_V:%.*]] = select i1 [[C]], i64 [[AND]], i64 [[CONV_1]]
; CHECK-NEXT: [[RED_NEXT]] = or i64 [[RED_NEXT_V]], [[RED]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOPS]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP13:%.*]] = phi i64 [ [[RED_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: ret i64 [[TMP13]]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[RED_NEXT]], %[[LOOPS]] ]
+; CHECK-NEXT: ret i64 [[RES]]
;
entry:
%ext.0 = sext i8 0 to i32
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index c8a408c580cdb..afd192246b7ef 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -48,25 +48,25 @@ define i32 @iv_used_widened_and_truncated(ptr %dst, i64 %N) #0 {
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[SCALAR_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[N_MOD_VF8:%.*]] = and i64 [[TMP0]], 3
+; CHECK-NEXT: [[N_MOD_VF8:%.*]] = and i64 [[TMP0]], 7
; CHECK-NEXT: [[N_VEC9:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF8]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[VEC_EPILOG_RESUME_VAL]] to i32
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <8 x i32> poison, i32 [[TMP6]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT13:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT12]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION14:%.*]] = add <8 x i32> [[BROADCAST_SPLAT13]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <8 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT11]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION13:%.*]] = add <8 x i32> [[BROADCAST_SPLAT12]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[INDEX15:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND16:%.*]] = phi <8 x i64> [ [[INDUCTION]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT20:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[VEC_IND17:%.*]] = phi <8 x i32> [ [[INDUCTION14]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT21:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[WIDE_GEP18:%.*]] = getelementptr { i32, [8 x i32] }, ptr [[DST]], <8 x i64> [[VEC_IND16]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[VEC_IND17]], <8 x ptr> align 8 [[WIDE_GEP18]], <8 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT19]] = add nuw i64 [[INDEX15]], 8
-; CHECK-NEXT: [[VEC_IND_NEXT20]] = add <8 x i64> [[VEC_IND16]], splat (i64 8)
-; CHECK-NEXT: [[VEC_IND_NEXT21]] = add <8 x i32> [[VEC_IND17]], splat (i32 8)
+; CHECK-NEXT: [[INDEX14:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND15:%.*]] = phi <8 x i64> [ [[INDUCTION]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT19:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VEC_IND16:%.*]] = phi <8 x i32> [ [[INDUCTION13]], %[[SCALAR_PH]] ], [ [[VEC_IND_NEXT20:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[WIDE_GEP17:%.*]] = getelementptr { i32, [8 x i32] }, ptr [[DST]], <8 x i64> [[VEC_IND15]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[VEC_IND16]], <8 x ptr> align 8 [[WIDE_GEP17]], <8 x i1> splat (i1 true))
+; CHECK-NEXT: [[INDEX_NEXT19]] = add nuw i64 [[INDEX14]], 8
+; CHECK-NEXT: [[VEC_IND_NEXT19]] = add <8 x i64> [[VEC_IND15]], splat (i64 8)
+; CHECK-NEXT: [[VEC_IND_NEXT20]] = add <8 x i32> [[VEC_IND16]], splat (i32 8)
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT19]], [[N_VEC9]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -495,7 +495,7 @@ define i32 @test_scalar_predicated_cost(i64 %x, i64 %y, ptr %A) #0 {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
-; CHECK-NEXT: br i1 false, label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK-NEXT: br i1 false, label %[[SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF25:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_PH]] ]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <4 x i64> poison, i64 [[Y]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/induction-cost.ll b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
index 8fbe1ba716cd8..c7bbd8f56934a 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-cost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost.*WIDEN-INDUCTION)|(induction instruction)" --filter-out-after "LV: Using user VF"
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "(Cost.*WIDEN-INDUCTION)|(induction instruction)" --filter-out-after "LV: Selecting VF"
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output %s 2>&1 | FileCheck %s
More information about the llvm-commits
mailing list