[llvm] [LoopVectorize] Further improve cost model for early exit loops (PR #126235)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 11 02:57:57 PDT 2025
https://github.com/david-arm updated https://github.com/llvm/llvm-project/pull/126235
>From 3a1f06c9986825ce08bb57b6d062b0efa6759ada Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Fri, 7 Feb 2025 11:44:01 +0000
Subject: [PATCH 1/6] [LoopVectorize] Further improve cost model for early exit
loops
Following on from #125058, this patch takes into account the
work done in the vector early exit block when assessing the
profitability of vectorising the loop. I have renamed
areRuntimeChecksProfitable to isOutsideLoopWorkProfitable and
we now pass in the early exit costs. As part of this, I have
added the ExtractFirstActive opcode to VPInstruction::computeCost.
It's worth pointing out that when we assess profitability of the
loop we calculate a minimum trip count and compare that against
the *maximum* trip count. However, since the loop has an early
exit the runtime trip count can still end up being less than the
minimum. Alternatively, we may never take the early exit at all
at runtime and so we have the opposite problem of over-estimating
the cost of the loop. The loop vectoriser cannot simultaneously
take two contradictory positions and so I feel the only sensible
thing to do is be conservative and assume the loop will be more
expensive than loops without early exits.
We may find in future that we need to adjust the cost according to
the probability of taking the early exit. This will become even
more important once we support multiple early exits. However, we
have to start somewhere and we can always revisit this later.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 56 +++++++++++++++----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 13 +++++
.../LoopVectorize/AArch64/early_exit_costs.ll | 47 ++++++++++++++++
.../AArch64/simple_early_exit.ll | 1 +
4 files changed, 105 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 02aa21e33bf4f..50bdbcbc799a8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10171,19 +10171,46 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
}
}
-static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
- VectorizationFactor &VF, Loop *L,
- PredicatedScalarEvolution &PSE,
- ScalarEpilogueLowering SEL,
- std::optional<unsigned> VScale) {
+static InstructionCost calculateEarlyExitCost(LoopVectorizationCostModel &CM,
+ VPlan &Plan, ElementCount VF) {
+ InstructionCost Cost = 0;
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(), CM,
+ CM.CostKind);
+ LLVM_DEBUG(
+ dbgs() << "Calculating cost of work in vector early exit block:\n");
+ for (auto *ExitVPBB : Plan.getExitBlocks()) {
+ for (auto *PredVPBB : ExitVPBB->getPredecessors())
+ if (PredVPBB != Plan.getMiddleBlock())
+ for (auto &R : *(cast<VPBasicBlock>(PredVPBB)))
+ Cost += R.cost(VF, CostCtx);
+ }
+ return Cost;
+}
+
+static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
+ VectorizationFactor &VF, Loop *L,
+ const TargetTransformInfo &TTI,
+ PredicatedScalarEvolution &PSE,
+ ScalarEpilogueLowering SEL,
+ std::optional<unsigned> VScale,
+ InstructionCost EarlyExitCost) {
InstructionCost CheckCost = Checks.getCost();
- if (!CheckCost.isValid())
+ if (!CheckCost.isValid() && !EarlyExitCost.isValid())
return false;
+ InstructionCost TotalCost = 0;
+ if (CheckCost.isValid())
+ TotalCost += CheckCost;
+
+ // Add on the cost of work required in the vector early exit block, if one
+ // exists.
+ if (EarlyExitCost.isValid())
+ TotalCost += EarlyExitCost;
+
// When interleaving only scalar and vector cost will be equal, which in turn
// would lead to a divide by 0. Fall back to hard threshold.
if (VF.Width.isScalar()) {
- if (CheckCost > VectorizeMemoryCheckThreshold) {
+ if (TotalCost > VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(
dbgs()
<< "LV: Interleaving only is not profitable due to runtime checks\n");
@@ -10227,7 +10254,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
// the computations are performed on doubles, not integers and the result
// is rounded up, hence we get an upper estimate of the TC.
unsigned IntVF = getEstimatedRuntimeVF(VF.Width, VScale);
- uint64_t RtC = *CheckCost.getValue();
+ uint64_t RtC = *TotalCost.getValue();
uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
@@ -10555,8 +10582,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// iteration count is low. However, setting the epilogue policy to
// `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
// with runtime checks. It's more effective to let
- // `areRuntimeChecksProfitable` determine if vectorization is beneficial
- // for the loop.
+ // `isOutsideLoopWorkProfitable` determine if vectorization is
+ // beneficial for the loop.
if (SEL != CM_ScalarEpilogueNotNeededUsePredicate)
SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
} else {
@@ -10651,12 +10678,17 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (VF.Width.isVector() || SelectedIC > 1)
Checks.create(L, *LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC);
+ InstructionCost EarlyExitCost = InstructionCost::getInvalid();
+ if (VF.Width.isVector() && LVL.hasUncountableEarlyExit())
+ EarlyExitCost =
+ calculateEarlyExitCost(CM, LVP.getPlanFor(VF.Width), VF.Width);
+
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
if (!ForceVectorization &&
- !areRuntimeChecksProfitable(Checks, VF, L, PSE, SEL,
- CM.getVScaleForTuning())) {
+ !isOutsideLoopWorkProfitable(Checks, VF, L, *TTI, PSE, SEL,
+ CM.getVScaleForTuning(), EarlyExitCost)) {
ORE->emit([&]() {
return OptimizationRemarkAnalysisAliasing(
DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a8b304271f0da..ee9e3733487a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -742,6 +742,19 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getArithmeticReductionCost(
Instruction::Or, cast<VectorType>(VecTy), std::nullopt, Ctx.CostKind);
}
+ case VPInstruction::ExtractFirstActive: {
+ // Calculate the cost of determining the lane index.
+ auto *PredTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(1)), VF);
+ IntrinsicCostAttributes Attrs(
+ Intrinsic::experimental_cttz_elts, Type::getInt64Ty(Ctx.LLVMCtx),
+ {PoisonValue::get(PredTy), ConstantInt::getTrue(Ctx.LLVMCtx)});
+ InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ // Add on the cost of extracting the element.
+ auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+ Cost += Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
+ Ctx.CostKind);
+ return Cost;
+ }
default:
// TODO: Compute cost other VPInstructions once the legacy cost model has
// been retired.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
new file mode 100644
index 0000000000000..d38d1f04b1e8d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; REQUIRES: asserts
+; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -disable-output \
+; RUN: -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefixes=CHECK
+
+target triple = "aarch64-unknown-linux-gnu"
+
+declare void @init_mem(ptr, i64);
+
+define i64 @same_exit_block_pre_inc_use1() #1 {
+; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
+; CHECK: LV: Selecting VF: vscale x 16
+; CHECK: Calculating cost of work in vector early exit block:
+; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:32
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %index2 = phi i64 [ %index2.next, %loop.inc ], [ 15, %entry ]
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %index2.next = add i64 %index2, 2
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %val1 = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ %val2 = phi i64 [ %index2, %loop ], [ 98, %loop.inc ]
+ %retval = add i64 %val1, %val2
+ ret i64 %retval
+}
+
+attributes #1 = { "target-features"="+sve" vscale_range(1,16) }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index 5f926db1131f6..c8aa3e92b1428 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -274,6 +274,7 @@ define i64 @loop_contains_safe_div() #1 {
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 12, i64 [[TMP12]])
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
>From 33272b00d27f1a0a20280032b35a30939d6f0e5d Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Thu, 27 Feb 2025 09:12:43 +0000
Subject: [PATCH 2/6] Address review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 58 +++++++++++--------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 +-
.../LoopVectorize/AArch64/early_exit_costs.ll | 43 +++++++++++++-
3 files changed, 78 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 50bdbcbc799a8..9c1179fe732fe 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10171,6 +10171,12 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
}
}
+/// For loops with uncountable early exits, find the cost of doing work when
+/// exiting the loop early, such as calculating the final exit values of
+/// variables used outside the loop.
+/// TODO: This is currently overly pessimistic because the loop may not take
+/// the early exit, but better to keep this conservative for now. In future,
+/// it might be possible to relax this by using branch probabilities.
static InstructionCost calculateEarlyExitCost(LoopVectorizationCostModel &CM,
VPlan &Plan, ElementCount VF) {
InstructionCost Cost = 0;
@@ -10179,37 +10185,44 @@ static InstructionCost calculateEarlyExitCost(LoopVectorizationCostModel &CM,
LLVM_DEBUG(
dbgs() << "Calculating cost of work in vector early exit block:\n");
for (auto *ExitVPBB : Plan.getExitBlocks()) {
- for (auto *PredVPBB : ExitVPBB->getPredecessors())
+ for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
+ // If the predecessor is not the middle.block, then it must be the
+ // vector.early.exit block, which may contain work to calculate the exit
+ // values of variables used outside the loop.
if (PredVPBB != Plan.getMiddleBlock())
for (auto &R : *(cast<VPBasicBlock>(PredVPBB)))
Cost += R.cost(VF, CostCtx);
+ }
}
return Cost;
}
+/// This function determines whether or not it's still profitable to vectorize
+/// the loop given the extra work we have to do outside of the loop:
+/// 1. Perform the runtime checks before entering the loop to ensure it's safe
+/// to vectorize.
+/// 2. In the case of loops with uncountable early exits, we may have to do
+/// extra work when exiting the loop early, such as calculating the final
+/// exit values of variables used outside the loop.
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
- VectorizationFactor &VF, Loop *L,
- const TargetTransformInfo &TTI,
+ VectorizationFactor &VF,
+ LoopVectorizationCostModel &CM,
PredicatedScalarEvolution &PSE,
- ScalarEpilogueLowering SEL,
- std::optional<unsigned> VScale,
- InstructionCost EarlyExitCost) {
- InstructionCost CheckCost = Checks.getCost();
- if (!CheckCost.isValid() && !EarlyExitCost.isValid())
+ VPlan &Plan,
+ ScalarEpilogueLowering SEL) {
+ InstructionCost TotalCost = Checks.getCost();
+ if (!TotalCost.isValid())
return false;
- InstructionCost TotalCost = 0;
- if (CheckCost.isValid())
- TotalCost += CheckCost;
-
// Add on the cost of work required in the vector early exit block, if one
// exists.
- if (EarlyExitCost.isValid())
- TotalCost += EarlyExitCost;
+ if (CM.Legal->hasUncountableEarlyExit())
+ TotalCost += calculateEarlyExitCost(CM, Plan, VF.Width);
// When interleaving only scalar and vector cost will be equal, which in turn
// would lead to a divide by 0. Fall back to hard threshold.
if (VF.Width.isScalar()) {
+ // TODO: Should we rename VectorizeMemoryCheckThreshold?
if (TotalCost > VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(
dbgs()
@@ -10236,7 +10249,9 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// The total cost of the vector loop is
// RtC + VecC * (TC / VF) + EpiC
// where
- // * RtC is the cost of the generated runtime checks
+ // * RtC is the cost of the generated runtime checks plus the cost of
+ // performing any additional work in the vector.early.exit block for loops
+ // with uncountable early exits.
// * VecC is the cost of a single vector iteration.
// * TC is the actual trip count of the loop
// * VF is the vectorization factor
@@ -10253,7 +10268,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
// the computations are performed on doubles, not integers and the result
// is rounded up, hence we get an upper estimate of the TC.
- unsigned IntVF = getEstimatedRuntimeVF(VF.Width, VScale);
+ unsigned IntVF = getEstimatedRuntimeVF(VF.Width, CM.getVScaleForTuning());
uint64_t RtC = *TotalCost.getValue();
uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
@@ -10281,7 +10296,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// Skip vectorization if the expected trip count is less than the minimum
// required trip count.
- if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
+ if (auto ExpectedTC = getSmallBestKnownTC(PSE, CM.TheLoop)) {
if (ElementCount::isKnownLT(ElementCount::getFixed(*ExpectedTC),
VF.MinProfitableTripCount)) {
LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
@@ -10678,17 +10693,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (VF.Width.isVector() || SelectedIC > 1)
Checks.create(L, *LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC);
- InstructionCost EarlyExitCost = InstructionCost::getInvalid();
- if (VF.Width.isVector() && LVL.hasUncountableEarlyExit())
- EarlyExitCost =
- calculateEarlyExitCost(CM, LVP.getPlanFor(VF.Width), VF.Width);
-
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
if (!ForceVectorization &&
- !isOutsideLoopWorkProfitable(Checks, VF, L, *TTI, PSE, SEL,
- CM.getVScaleForTuning(), EarlyExitCost)) {
+ !isOutsideLoopWorkProfitable(Checks, VF, CM, PSE,
+ LVP.getPlanFor(VF.Width), SEL)) {
ORE->emit([&]() {
return OptimizationRemarkAnalysisAliasing(
DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ee9e3733487a0..91c0ec997877e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -745,9 +745,9 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
case VPInstruction::ExtractFirstActive: {
// Calculate the cost of determining the lane index.
auto *PredTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(1)), VF);
- IntrinsicCostAttributes Attrs(
- Intrinsic::experimental_cttz_elts, Type::getInt64Ty(Ctx.LLVMCtx),
- {PoisonValue::get(PredTy), ConstantInt::getTrue(Ctx.LLVMCtx)});
+ IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
+ Type::getInt64Ty(Ctx.LLVMCtx),
+ {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
// Add on the cost of extracting the element.
auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
index d38d1f04b1e8d..a0359e05d9c43 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
@@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu"
declare void @init_mem(ptr, i64);
-define i64 @same_exit_block_pre_inc_use1() #1 {
-; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
+define i64 @same_exit_block_pre_inc_use1_sve() #1 {
+; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_sve'
; CHECK: LV: Selecting VF: vscale x 16
; CHECK: Calculating cost of work in vector early exit block:
; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
@@ -44,4 +44,43 @@ loop.end:
ret i64 %retval
}
+define i64 @same_exit_block_pre_inc_use1_nosve() {
+; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_nosve'
+; CHECK: LV: Selecting VF: 16
+; CHECK: Calculating cost of work in vector early exit block:
+; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:176
+; CHECK-NEXT: LV: Vectorization is not beneficial: expected trip count < minimum profitable VF (64 < 176)
+; CHECK-NEXT: LV: Too many memory checks needed.
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %index2 = phi i64 [ %index2.next, %loop.inc ], [ 15, %entry ]
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %index2.next = add i64 %index2, 2
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %val1 = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ %val2 = phi i64 [ %index2, %loop ], [ 98, %loop.inc ]
+ %retval = add i64 %val1, %val2
+ ret i64 %retval
+}
+
attributes #1 = { "target-features"="+sve" vscale_range(1,16) }
>From 0e082a2eb2e20318583a4d6f734ef8513914ce1d Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Tue, 4 Mar 2025 14:45:59 +0000
Subject: [PATCH 3/6] Address review comments
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 10 ++++------
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 ++---
.../AArch64/low_trip_memcheck_cost.ll | 14 +++++++-------
3 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9c1179fe732fe..595ff05444b2f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10190,8 +10190,7 @@ static InstructionCost calculateEarlyExitCost(LoopVectorizationCostModel &CM,
// vector.early.exit block, which may contain work to calculate the exit
// values of variables used outside the loop.
if (PredVPBB != Plan.getMiddleBlock())
- for (auto &R : *(cast<VPBasicBlock>(PredVPBB)))
- Cost += R.cost(VF, CostCtx);
+ Cost += PredVPBB->cost(VF, CostCtx);
}
}
return Cost;
@@ -10214,10 +10213,9 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
if (!TotalCost.isValid())
return false;
- // Add on the cost of work required in the vector early exit block, if one
- // exists.
- if (CM.Legal->hasUncountableEarlyExit())
- TotalCost += calculateEarlyExitCost(CM, Plan, VF.Width);
+ // Add on the cost of any work required in the vector early exit block, if
+ // one exists.
+ TotalCost += calculateEarlyExitCost(CM, Plan, VF.Width);
// When interleaving only scalar and vector cost will be equal, which in turn
// would lead to a divide by 0. Fall back to hard threshold.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 91c0ec997877e..8da7ea3a8ca70 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -751,9 +751,8 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
// Add on the cost of extracting the element.
auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
- Cost += Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
- Ctx.CostKind);
- return Cost;
+ return Cost + Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
+ Ctx.CostKind);
}
default:
// TODO: Compute cost other VPInstructions once the legacy cost model has
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
index 611b980999bfe..df1c639911cb0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
@@ -8,7 +8,7 @@ define void @no_outer_loop(ptr nocapture noundef %a, ptr nocapture noundef reado
; CHECK: Calculating cost of runtime checks:
; CHECK-NOT: We expect runtime memory checks to be hoisted out of the outer loop.
; CHECK: Total cost of runtime checks: 4
-; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %inner.loop
@@ -34,7 +34,7 @@ define void @outer_no_tc(ptr nocapture noundef %a, ptr nocapture noundef readonl
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 3
; CHECK: Total cost of runtime checks: 3
-; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -71,7 +71,7 @@ define void @outer_known_tc3(ptr nocapture noundef %a, ptr nocapture noundef rea
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
-; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -108,7 +108,7 @@ define void @outer_known_tc64(ptr nocapture noundef %a, ptr nocapture noundef re
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 1
; CHECK: Total cost of runtime checks: 1
-; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -145,7 +145,7 @@ define void @outer_pgo_3(ptr nocapture noundef %a, ptr nocapture noundef readonl
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
-; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -182,7 +182,7 @@ define void @outer_pgo_minus1(ptr nocapture noundef %a, ptr nocapture noundef re
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 1
; CHECK: Total cost of runtime checks: 1
-; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -219,7 +219,7 @@ define void @outer_known_tc3_full_range_checks(ptr nocapture noundef %dst, ptr n
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
-; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:4
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:4
entry:
br label %outer.loop
>From b22497f0aba98df79ee9091af0333bb2883b7cc9 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Wed, 5 Mar 2025 15:47:03 +0000
Subject: [PATCH 4/6] Fix test after rebase
---
llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index c8aa3e92b1428..ab4738bf2901b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -274,7 +274,7 @@ define i64 @loop_contains_safe_div() #1 {
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 4
-; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 12, i64 [[TMP12]])
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP12]])
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
>From 4afbfd08a23ebc5dbb6b526f05728c3e036d5bf9 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Mon, 10 Mar 2025 12:58:35 +0000
Subject: [PATCH 5/6] Address review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 32 ++++++++++---------
.../LoopVectorize/AArch64/early_exit_costs.ll | 4 +--
2 files changed, 19 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 595ff05444b2f..9e65636865b2c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10177,20 +10177,19 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
/// TODO: This is currently overly pessimistic because the loop may not take
/// the early exit, but better to keep this conservative for now. In future,
/// it might be possible to relax this by using branch probabilities.
-static InstructionCost calculateEarlyExitCost(LoopVectorizationCostModel &CM,
+static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
VPlan &Plan, ElementCount VF) {
InstructionCost Cost = 0;
- VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(), CM,
- CM.CostKind);
- LLVM_DEBUG(
- dbgs() << "Calculating cost of work in vector early exit block:\n");
for (auto *ExitVPBB : Plan.getExitBlocks()) {
for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
// If the predecessor is not the middle.block, then it must be the
// vector.early.exit block, which may contain work to calculate the exit
// values of variables used outside the loop.
- if (PredVPBB != Plan.getMiddleBlock())
+ if (PredVPBB != Plan.getMiddleBlock()) {
+ LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
+ << PredVPBB->getName() << ":\n");
Cost += PredVPBB->cost(VF, CostCtx);
+ }
}
}
return Cost;
@@ -10204,18 +10203,18 @@ static InstructionCost calculateEarlyExitCost(LoopVectorizationCostModel &CM,
/// extra work when exiting the loop early, such as calculating the final
/// exit values of variables used outside the loop.
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
- VectorizationFactor &VF,
- LoopVectorizationCostModel &CM,
+ VectorizationFactor &VF, Loop *L,
PredicatedScalarEvolution &PSE,
- VPlan &Plan,
- ScalarEpilogueLowering SEL) {
+ VPCostContext &CostCtx, VPlan &Plan,
+ ScalarEpilogueLowering SEL,
+ std::optional<unsigned> VScale) {
InstructionCost TotalCost = Checks.getCost();
if (!TotalCost.isValid())
return false;
// Add on the cost of any work required in the vector early exit block, if
// one exists.
- TotalCost += calculateEarlyExitCost(CM, Plan, VF.Width);
+ TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF.Width);
// When interleaving only scalar and vector cost will be equal, which in turn
// would lead to a divide by 0. Fall back to hard threshold.
@@ -10266,7 +10265,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
// the computations are performed on doubles, not integers and the result
// is rounded up, hence we get an upper estimate of the TC.
- unsigned IntVF = getEstimatedRuntimeVF(VF.Width, CM.getVScaleForTuning());
+ unsigned IntVF = getEstimatedRuntimeVF(VF.Width, VScale);
uint64_t RtC = *TotalCost.getValue();
uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
@@ -10294,7 +10293,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
// Skip vectorization if the expected trip count is less than the minimum
// required trip count.
- if (auto ExpectedTC = getSmallBestKnownTC(PSE, CM.TheLoop)) {
+ if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
if (ElementCount::isKnownLT(ElementCount::getFixed(*ExpectedTC),
VF.MinProfitableTripCount)) {
LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
@@ -10694,9 +10693,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(),
+ CM, CM.CostKind);
if (!ForceVectorization &&
- !isOutsideLoopWorkProfitable(Checks, VF, CM, PSE,
- LVP.getPlanFor(VF.Width), SEL)) {
+ !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
+ LVP.getPlanFor(VF.Width), SEL,
+ CM.getVScaleForTuning())) {
ORE->emit([&]() {
return OptimizationRemarkAnalysisAliasing(
DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
index a0359e05d9c43..55c6c43b6306a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
@@ -10,7 +10,7 @@ declare void @init_mem(ptr, i64);
define i64 @same_exit_block_pre_inc_use1_sve() #1 {
; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_sve'
; CHECK: LV: Selecting VF: vscale x 16
-; CHECK: Calculating cost of work in vector early exit block:
+; CHECK: Calculating cost of work in exit block vector.early.exit
; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
; CHECK: LV: Minimum required TC for runtime checks to be profitable:32
@@ -47,7 +47,7 @@ loop.end:
define i64 @same_exit_block_pre_inc_use1_nosve() {
; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_nosve'
; CHECK: LV: Selecting VF: 16
-; CHECK: Calculating cost of work in vector early exit block:
+; CHECK: Calculating cost of work in exit block vector.early.exit
; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
; CHECK: LV: Minimum required TC for runtime checks to be profitable:176
>From 08c1f5b6371acdd3797cdbb6016683b32b7e5dd7 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood at arm.com>
Date: Tue, 11 Mar 2025 09:56:41 +0000
Subject: [PATCH 6/6] Address review comment
---
.../AArch64/low_trip_memcheck_cost.ll | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
index df1c639911cb0..611b980999bfe 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
@@ -8,7 +8,7 @@ define void @no_outer_loop(ptr nocapture noundef %a, ptr nocapture noundef reado
; CHECK: Calculating cost of runtime checks:
; CHECK-NOT: We expect runtime memory checks to be hoisted out of the outer loop.
; CHECK: Total cost of runtime checks: 4
-; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %inner.loop
@@ -34,7 +34,7 @@ define void @outer_no_tc(ptr nocapture noundef %a, ptr nocapture noundef readonl
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 3
; CHECK: Total cost of runtime checks: 3
-; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -71,7 +71,7 @@ define void @outer_known_tc3(ptr nocapture noundef %a, ptr nocapture noundef rea
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
-; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -108,7 +108,7 @@ define void @outer_known_tc64(ptr nocapture noundef %a, ptr nocapture noundef re
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 1
; CHECK: Total cost of runtime checks: 1
-; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -145,7 +145,7 @@ define void @outer_pgo_3(ptr nocapture noundef %a, ptr nocapture noundef readonl
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
-; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -182,7 +182,7 @@ define void @outer_pgo_minus1(ptr nocapture noundef %a, ptr nocapture noundef re
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 1
; CHECK: Total cost of runtime checks: 1
-; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -219,7 +219,7 @@ define void @outer_known_tc3_full_range_checks(ptr nocapture noundef %dst, ptr n
; CHECK: Calculating cost of runtime checks:
; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 6 to 2
; CHECK: Total cost of runtime checks: 2
-; CHECK: LV: Minimum required TC for runtime checks to be profitable:4
+; CHECK-NEXT: LV: Minimum required TC for runtime checks to be profitable:4
entry:
br label %outer.loop
More information about the llvm-commits
mailing list