[llvm] [LoopVectorize] Enhance Vectorization decisions for predicate tail-folded loops with low trip counts (PR #69588)
Igor Kirillov via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 30 04:28:56 PDT 2023
https://github.com/igogo-x86 updated https://github.com/llvm/llvm-project/pull/69588
>From 6d85c024a0e2bf27b6881972b4b0b878f54b3c2e Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Mon, 16 Oct 2023 12:56:55 +0000
Subject: [PATCH 1/2] [LoopVectorize] Enhance Vectorization decisions for
predicate tail-folded loops with low trip counts
* Avoid using `CM_ScalarEpilogueNotAllowedLowTripLoop` for loops known
to be predicate tail-folded, delegating to `areRuntimeChecksProfitable`
to decide on the profitability of vectorizing loops with runtime checks.
* Update the `areRuntimeChecksProfitable` function to consider the
`ScalarEpilogueLowering` setting when assessing vectorization of a loop.
With this patch, we can make more informed decisions for loops with low
trip counts, especially when leveraging Profile-Guided Optimization (PGO)
data.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 24 +++++++++++++------
1 file changed, 17 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 16c761a91ff2326..4f547886f602534 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9785,7 +9785,8 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
VectorizationFactor &VF,
std::optional<unsigned> VScale, Loop *L,
- ScalarEvolution &SE) {
+ ScalarEvolution &SE,
+ ScalarEpilogueLowering SEL) {
InstructionCost CheckCost = Checks.getCost();
if (!CheckCost.isValid())
return false;
@@ -9855,11 +9856,13 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
// RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
double MinTC2 = RtC * 10 / ScalarC;
- // Now pick the larger minimum. If it is not a multiple of VF, choose the
- // next closest multiple of VF. This should partly compensate for ignoring
- // the epilogue cost.
+ // Now pick the larger minimum. If it is not a multiple of VF and a scalar
+ // epilogue is allowed, choose the next closest multiple of VF. This should
+ // partly compensate for ignoring the epilogue cost.
uint64_t MinTC = std::ceil(std::max(MinTC1, MinTC2));
- VF.MinProfitableTripCount = ElementCount::getFixed(alignTo(MinTC, IntVF));
+ if (SEL == CM_ScalarEpilogueAllowed)
+ MinTC = alignTo(MinTC, IntVF);
+ VF.MinProfitableTripCount = ElementCount::getFixed(MinTC);
LLVM_DEBUG(
dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
@@ -9979,7 +9982,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
else {
if (*ExpectedTC > TTI->getMinTripCountTailFoldingThreshold()) {
LLVM_DEBUG(dbgs() << "\n");
- SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
+ // Predicate tail-folded loops are efficient even when the loop
+ // iteration count is low. However, setting the epilogue policy to
+ // `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
+ // with runtime checks. It's more effective to let
+ // `areRuntimeChecksProfitable` determine if vectorization is beneficial
+ // for the loop.
+ if (SEL != CM_ScalarEpilogueNotNeededUsePredicate)
+ SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
} else {
LLVM_DEBUG(dbgs() << " But the target considers the trip count too "
"small to consider vectorizing.\n");
@@ -10074,7 +10084,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
if (!ForceVectorization &&
!areRuntimeChecksProfitable(Checks, VF, getVScaleForTuning(L, *TTI), L,
- *PSE.getSE())) {
+ *PSE.getSE(), SEL)) {
ORE->emit([&]() {
return OptimizationRemarkAnalysisAliasing(
DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
>From dce7f5087540b16c4af2662484749d4aff402671 Mon Sep 17 00:00:00 2001
From: Igor Kirillov <igor.kirillov at arm.com>
Date: Tue, 24 Oct 2023 14:31:00 +0000
Subject: [PATCH 2/2] Add a test showing different behaviour with PGO profile
data when different tail-folding strategy is chosen
---
.../runtime-check-trip-count-decisions.ll | 108 ++++++++++++++++++
1 file changed, 108 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
new file mode 100644
index 000000000000000..39ef5baa5b01905
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
@@ -0,0 +1,108 @@
+; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck --check-prefixes=CHECK,PREDICATED %s
+; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -prefer-predicate-over-epilogue=scalar-epilogue -S | FileCheck --check-prefixes=CHECK,SCALAR %s
+
+; This file contains the same function but with different trip-count PGO hints
+
+; The function is vectorized if there are no trip-count hints
+define i32 @foo_no_trip_count(ptr %a, ptr %b, ptr %c, i32 %bound) {
+; CHECK-LABEL: @foo_no_trip_count(
+; PREDICATED: vector.body
+; SCALAR: vector.body
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %idx = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %a.index = getelementptr inbounds [32 x i8], ptr %a, i32 0, i32 %idx
+ %0 = load i8, ptr %a.index, align 1
+ %b.index = getelementptr inbounds [32 x i8], ptr %b, i32 0, i32 %idx
+ %1 = load i8, ptr %b.index, align 1
+ %2 = add i8 %0, %1
+ %c.index = getelementptr inbounds [32 x i8], ptr %c, i32 0, i32 %idx
+ store i8 %2, ptr %c.index, align 1
+ %inc = add nsw i32 %idx, 1
+ %exitcond = icmp eq i32 %idx, %bound
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body
+ ret i32 0
+}
+
+; If trip-count is equal to 4, the function is not vectorised
+define i32 @foo_low_trip_count(ptr %a, ptr %b, ptr %c, i32 %bound) {
+; CHECK-LABEL: @foo_low_trip_count(
+; PREDICATED-NOT: vector.body
+; SCALAR-NOT: vector.body
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %idx = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %a.index = getelementptr inbounds [32 x i8], ptr %a, i32 0, i32 %idx
+ %0 = load i8, ptr %a.index, align 1
+ %b.index = getelementptr inbounds [32 x i8], ptr %b, i32 0, i32 %idx
+ %1 = load i8, ptr %b.index, align 1
+ %2 = add i8 %0, %1
+ %c.index = getelementptr inbounds [32 x i8], ptr %c, i32 0, i32 %idx
+ store i8 %2, ptr %c.index, align 1
+ %inc = add nsw i32 %idx, 1
+ %exitcond = icmp eq i32 %idx, %bound
+ br i1 %exitcond, label %for.end, label %for.body, !prof !0
+
+for.end: ; preds = %for.body
+ ret i32 0
+}
+
+; If trip-count is equal to 10, the function is vectorised when predicated tail folding is chosen
+define i32 @foo_mid_trip_count(ptr %a, ptr %b, ptr %c, i32 %bound) {
+; CHECK-LABEL: @foo_mid_trip_count(
+; PREDICATED: vector.body
+; SCALAR-NOT: vector.body
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %idx = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %a.index = getelementptr inbounds [32 x i8], ptr %a, i32 0, i32 %idx
+ %0 = load i8, ptr %a.index, align 1
+ %b.index = getelementptr inbounds [32 x i8], ptr %b, i32 0, i32 %idx
+ %1 = load i8, ptr %b.index, align 1
+ %2 = add i8 %0, %1
+ %c.index = getelementptr inbounds [32 x i8], ptr %c, i32 0, i32 %idx
+ store i8 %2, ptr %c.index, align 1
+ %inc = add nsw i32 %idx, 1
+ %exitcond = icmp eq i32 %idx, %bound
+ br i1 %exitcond, label %for.end, label %for.body, !prof !1
+
+for.end: ; preds = %for.body
+ ret i32 0
+}
+
+; If trip-count is equal to 40, the function is always vectorised
+define i32 @foo_high_trip_count(ptr %a, ptr %b, ptr %c, i32 %bound) {
+; CHECK-LABEL: @foo_high_trip_count(
+; PREDICATED: vector.body
+; SCALAR: vector.body
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %idx = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %a.index = getelementptr inbounds [32 x i8], ptr %a, i32 0, i32 %idx
+ %0 = load i8, ptr %a.index, align 1
+ %b.index = getelementptr inbounds [32 x i8], ptr %b, i32 0, i32 %idx
+ %1 = load i8, ptr %b.index, align 1
+ %2 = add i8 %0, %1
+ %c.index = getelementptr inbounds [32 x i8], ptr %c, i32 0, i32 %idx
+ store i8 %2, ptr %c.index, align 1
+ %inc = add nsw i32 %idx, 1
+ %exitcond = icmp eq i32 %idx, %bound
+ br i1 %exitcond, label %for.end, label %for.body, !prof !2
+
+for.end: ; preds = %for.body
+ ret i32 0
+}
+
+!0 = !{!"branch_weights", i32 10, i32 30}
+!1 = !{!"branch_weights", i32 10, i32 90}
+!2 = !{!"branch_weights", i32 10, i32 390}
More information about the llvm-commits
mailing list