[llvm] [LoopVectorize] Further improve cost model for early exit loops (PR #126235)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 7 03:53:26 PST 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: David Sherwood (david-arm)
<details>
<summary>Changes</summary>
Following on from #<!-- -->125058, this patch takes into account the
work done in the vector early exit block when assessing the
profitability of vectorising the loop. I have renamed
areRuntimeChecksProfitable to isOutsideLoopWorkProfitable and
we now pass in the early exit costs. As part of this, I have
added the ExtractFirstActive opcode to VPInstruction::computeCost.
It's worth pointing out that when we assess profitability of the
loop we calculate a minimum trip count and compare that against
the *maximum* trip count. However, since the loop has an early
exit the runtime trip count can still end up being less than the
minimum. Alternatively, we may never take the early exit at all
at runtime and so we have the opposite problem of over-estimating
the cost of the loop. The loop vectoriser cannot simultaneously
take two contradictory positions and so I feel the only sensible
thing to do is be conservative and assume the loop will be more
expensive than loops without early exits.
We may find in future that we need to adjust the cost according to
the probability of taking the early exit. This will become even
more important once we support multiple early exits. However, we
have to start somewhere and we can always revisit this later.
---
Full diff: https://github.com/llvm/llvm-project/pull/126235.diff
4 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+44-12)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+13)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll (+47)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll (+1)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6ad44259ccdf6e..7538dbe07a6466 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10077,19 +10077,46 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
}
}
-static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
- VectorizationFactor &VF, Loop *L,
- PredicatedScalarEvolution &PSE,
- ScalarEpilogueLowering SEL,
- std::optional<unsigned> VScale) {
+static InstructionCost calculateEarlyExitCost(LoopVectorizationCostModel &CM,
+ VPlan &Plan, ElementCount VF) {
+ InstructionCost Cost = 0;
+ VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(), CM,
+ CM.CostKind);
+ LLVM_DEBUG(
+ dbgs() << "Calculating cost of work in vector early exit block:\n");
+ for (auto *ExitVPBB : Plan.getExitBlocks()) {
+ for (auto *PredVPBB : ExitVPBB->getPredecessors())
+ if (PredVPBB != Plan.getMiddleBlock())
+ for (auto &R : *(cast<VPBasicBlock>(PredVPBB)))
+ Cost += R.cost(VF, CostCtx);
+ }
+ return Cost;
+}
+
+static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
+ VectorizationFactor &VF, Loop *L,
+ const TargetTransformInfo &TTI,
+ PredicatedScalarEvolution &PSE,
+ ScalarEpilogueLowering SEL,
+ std::optional<unsigned> VScale,
+ InstructionCost EarlyExitCost) {
InstructionCost CheckCost = Checks.getCost();
- if (!CheckCost.isValid())
+ if (!CheckCost.isValid() && !EarlyExitCost.isValid())
return false;
+ InstructionCost TotalCost = 0;
+ if (CheckCost.isValid())
+ TotalCost += CheckCost;
+
+ // Add on the cost of work required in the vector early exit block, if one
+ // exists.
+ if (EarlyExitCost.isValid())
+ TotalCost += EarlyExitCost;
+
// When interleaving only scalar and vector cost will be equal, which in turn
// would lead to a divide by 0. Fall back to hard threshold.
if (VF.Width.isScalar()) {
- if (CheckCost > VectorizeMemoryCheckThreshold) {
+ if (TotalCost > VectorizeMemoryCheckThreshold) {
LLVM_DEBUG(
dbgs()
<< "LV: Interleaving only is not profitable due to runtime checks\n");
@@ -10132,7 +10159,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
// the computations are performed on doubles, not integers and the result
// is rounded up, hence we get an upper estimate of the TC.
unsigned IntVF = getEstimatedRuntimeVF(VF.Width, VScale);
- uint64_t RtC = *CheckCost.getValue();
+ uint64_t RtC = *TotalCost.getValue();
uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
@@ -10468,8 +10495,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// iteration count is low. However, setting the epilogue policy to
// `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
// with runtime checks. It's more effective to let
- // `areRuntimeChecksProfitable` determine if vectorization is beneficial
- // for the loop.
+ // `isOutsideLoopWorkProfitable` determine if vectorization is
+ // beneficial for the loop.
if (SEL != CM_ScalarEpilogueNotNeededUsePredicate)
SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
} else {
@@ -10564,12 +10591,17 @@ bool LoopVectorizePass::processLoop(Loop *L) {
if (VF.Width.isVector() || SelectedIC > 1)
Checks.create(L, *LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC);
+ InstructionCost EarlyExitCost = InstructionCost::getInvalid();
+ if (VF.Width.isVector() && LVL.hasUncountableEarlyExit())
+ EarlyExitCost =
+ calculateEarlyExitCost(CM, LVP.getPlanFor(VF.Width), VF.Width);
+
// Check if it is profitable to vectorize with runtime checks.
bool ForceVectorization =
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
if (!ForceVectorization &&
- !areRuntimeChecksProfitable(Checks, VF, L, PSE, SEL,
- CM.getVScaleForTuning())) {
+ !isOutsideLoopWorkProfitable(Checks, VF, L, *TTI, PSE, SEL,
+ CM.getVScaleForTuning(), EarlyExitCost)) {
ORE->emit([&]() {
return OptimizationRemarkAnalysisAliasing(
DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a1b78fa244e17a..6ca82ef99a23bc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -726,6 +726,19 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
return Ctx.TTI.getArithmeticReductionCost(
Instruction::Or, cast<VectorType>(VecTy), std::nullopt, Ctx.CostKind);
}
+ case VPInstruction::ExtractFirstActive: {
+ // Calculate the cost of determining the lane index.
+ auto *PredTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(1)), VF);
+ IntrinsicCostAttributes Attrs(
+ Intrinsic::experimental_cttz_elts, Type::getInt64Ty(Ctx.LLVMCtx),
+ {PoisonValue::get(PredTy), ConstantInt::getTrue(Ctx.LLVMCtx)});
+ InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ // Add on the cost of extracting the element.
+ auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+ Cost += Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
+ Ctx.CostKind);
+ return Cost;
+ }
default:
// TODO: Fill out other opcodes!
return 0;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
new file mode 100644
index 00000000000000..d38d1f04b1e8de
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; REQUIRES: asserts
+; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -disable-output \
+; RUN: -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefixes=CHECK
+
+target triple = "aarch64-unknown-linux-gnu"
+
+declare void @init_mem(ptr, i64);
+
+define i64 @same_exit_block_pre_inc_use1() #1 {
+; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1'
+; CHECK: LV: Selecting VF: vscale x 16
+; CHECK: Calculating cost of work in vector early exit block:
+; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:32
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %index2 = phi i64 [ %index2.next, %loop.inc ], [ 15, %entry ]
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %index2.next = add i64 %index2, 2
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %val1 = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ %val2 = phi i64 [ %index2, %loop ], [ 98, %loop.inc ]
+ %retval = add i64 %val1, %val2
+ ret i64 %retval
+}
+
+attributes #1 = { "target-features"="+sve" vscale_range(1,16) }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index b439b64e829e5f..1b4172c58a034f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -274,6 +274,7 @@ define i64 @loop_contains_safe_div() #1 {
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 12, i64 [[TMP12]])
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
``````````
</details>
https://github.com/llvm/llvm-project/pull/126235
More information about the llvm-commits
mailing list