[llvm] [LoopPeel] Use loop guards when checking if last iter can be peeled. (PR #142605)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 3 13:56:52 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/142605
>From 9306a03f15e5a9970133c8e18b88df95d3da19e4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 3 Jun 2025 12:18:28 +0100
Subject: [PATCH 1/2] [LoopPeel] Use loop guards when checking if last iter can
be peeled.
Apply loop guards to BTC before checking if the last iteration should be
peeled off. This also adds an assert to make sure applying the guards
does not pessimize the results. I checked on a large test set and it did
not trigger there, but it adds an additional guard to catch potential
cases where loop-guards pessimize results.
Peels ~15% more loops.
---
llvm/lib/Transforms/Utils/LoopPeel.cpp | 27 +++++++++++-------
.../peel-last-iteration-with-guards.ll | 28 ++++++++++++++++---
2 files changed, 41 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index bd025fddd0cf7..564c2298f6f2f 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -357,17 +357,14 @@ bool llvm::canPeelLastIteration(const Loop &L, ScalarEvolution &SE) {
m_scev_AffineAddRec(m_SCEV(), m_scev_One(), m_SpecificLoop(&L)));
}
-/// Returns true if the last iteration can be peeled off and the condition (Pred
-/// LeftAR, RightSCEV) is known at the last iteration and the inverse condition
-/// is known at the second-to-last.
+/// Returns true if the last iteration should be peeled off, i.e. the condition
+/// (Pred LeftAR, RightSCEV) is known at the last iteration and the inverse
+/// condition is known at the second-to-last.
static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
const SCEVAddRecExpr *LeftAR,
- const SCEV *RightSCEV, ScalarEvolution &SE,
+ const SCEV *RightSCEV, const SCEV *BTC,
+ ScalarEvolution &SE,
const TargetTransformInfo &TTI) {
- if (!canPeelLastIteration(L, SE))
- return false;
-
- const SCEV *BTC = SE.getBackedgeTakenCount(&L);
SCEVExpander Expander(SE, L.getHeader()->getDataLayout(), "loop-peel");
if (!SE.isKnownNonZero(BTC) &&
Expander.isHighCostExpansion(BTC, &L, SCEVCheapExpansionBudget, &TTI,
@@ -377,7 +374,6 @@ static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
const SCEV *ValAtLastIter = LeftAR->evaluateAtIteration(BTC, SE);
const SCEV *ValAtSecondToLastIter = LeftAR->evaluateAtIteration(
SE.getMinusSCEV(BTC, SE.getOne(BTC->getType())), SE);
-
return SE.isKnownPredicate(ICmpInst::getInversePredicate(Pred), ValAtLastIter,
RightSCEV) &&
SE.isKnownPredicate(Pred, ValAtSecondToLastIter, RightSCEV);
@@ -484,8 +480,19 @@ countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
const SCEV *Step = LeftAR->getStepRecurrence(SE);
if (!PeelWhilePredicateIsKnown(NewPeelCount, IterVal, RightSCEV, Step,
Pred)) {
- if (shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, SE, TTI))
+ if (!canPeelLastIteration(L, SE))
+ return;
+
+ const SCEV *BTC = SE.getBackedgeTakenCount(&L);
+ auto Guards = ScalarEvolution::LoopGuards::collect(&L, SE);
+ if (shouldPeelLastIteration(L, Pred, LeftAR,
+ SE.applyLoopGuards(RightSCEV, Guards),
+ SE.applyLoopGuards(BTC, Guards), SE, TTI))
DesiredPeelCountLast = 1;
+ else
+ assert(!shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, BTC, SE,
+ TTI) &&
+ "loop guards pessimized result");
return;
}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll
index af07a97131322..824e23fcf3e6e 100644
--- a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll
+++ b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll
@@ -13,14 +13,33 @@ define void @peel_with_guard_known_nonnegative_1(i32 %n) {
; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64
; CHECK-NEXT: [[N_1:%.*]] = add i32 [[N]], 1
; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N_1]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i64 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PH_SPLIT:.*]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN:.*]]
+; CHECK: [[PH_SPLIT]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[PH_SPLIT]] ], [ [[IV_NEXT1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], 1
+; CHECK-NEXT: [[EC1:%.*]] = icmp eq i64 [[IV_NEXT1]], [[TMP2]]
+; CHECK-NEXT: br i1 [[EC1]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i64 [ [[IV_NEXT1]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_LOOPEXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_LOOPEXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[PH]] ], [ [[DOTPH]], %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[IV]], [[N_EXT]]
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[C]], i32 10, i32 20
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT_PEEL_NEXT:.*]], label %[[EXIT_LOOPEXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_LOOPEXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
@@ -137,7 +156,7 @@ define void @peel_with_guard2(i32 %n) {
; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 1
; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[N]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[TMP2]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK: [[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]]:
; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP_LATCH]] ]
; CHECK-NEXT: br label %[[EXIT_LOOPEXIT_PEEL_BEGIN]]
@@ -188,4 +207,5 @@ exit:
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
;.
>From c7deb118f0a724c2171e1c45aee42b0ffc014993 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 3 Jun 2025 21:42:39 +0100
Subject: [PATCH 2/2] !fixup remove assert.
---
llvm/lib/Transforms/Utils/LoopPeel.cpp | 30 +++++++++++---------------
1 file changed, 13 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index 564c2298f6f2f..cd6fdcce7d2a6 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -357,23 +357,30 @@ bool llvm::canPeelLastIteration(const Loop &L, ScalarEvolution &SE) {
m_scev_AffineAddRec(m_SCEV(), m_scev_One(), m_SpecificLoop(&L)));
}
-/// Returns true if the last iteration should be peeled off, i.e. the condition
-/// (Pred LeftAR, RightSCEV) is known at the last iteration and the inverse
-/// condition is known at the second-to-last.
+/// Returns true if the last iteration can be peeled off and the condition (Pred
+/// LeftAR, RightSCEV) is known at the last iteration and the inverse condition
+/// is known at the second-to-last.
static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
const SCEVAddRecExpr *LeftAR,
- const SCEV *RightSCEV, const SCEV *BTC,
- ScalarEvolution &SE,
+ const SCEV *RightSCEV, ScalarEvolution &SE,
const TargetTransformInfo &TTI) {
+ if (!canPeelLastIteration(L, SE))
+ return false;
+
+ const SCEV *BTC = SE.getBackedgeTakenCount(&L);
SCEVExpander Expander(SE, L.getHeader()->getDataLayout(), "loop-peel");
if (!SE.isKnownNonZero(BTC) &&
Expander.isHighCostExpansion(BTC, &L, SCEVCheapExpansionBudget, &TTI,
L.getLoopPredecessor()->getTerminator()))
return false;
+ auto Guards = ScalarEvolution::LoopGuards::collect(&L, SE);
+ BTC = SE.applyLoopGuards(BTC, Guards);
+ RightSCEV = SE.applyLoopGuards(RightSCEV, Guards);
const SCEV *ValAtLastIter = LeftAR->evaluateAtIteration(BTC, SE);
const SCEV *ValAtSecondToLastIter = LeftAR->evaluateAtIteration(
SE.getMinusSCEV(BTC, SE.getOne(BTC->getType())), SE);
+
return SE.isKnownPredicate(ICmpInst::getInversePredicate(Pred), ValAtLastIter,
RightSCEV) &&
SE.isKnownPredicate(Pred, ValAtSecondToLastIter, RightSCEV);
@@ -480,19 +487,8 @@ countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
const SCEV *Step = LeftAR->getStepRecurrence(SE);
if (!PeelWhilePredicateIsKnown(NewPeelCount, IterVal, RightSCEV, Step,
Pred)) {
- if (!canPeelLastIteration(L, SE))
- return;
-
- const SCEV *BTC = SE.getBackedgeTakenCount(&L);
- auto Guards = ScalarEvolution::LoopGuards::collect(&L, SE);
- if (shouldPeelLastIteration(L, Pred, LeftAR,
- SE.applyLoopGuards(RightSCEV, Guards),
- SE.applyLoopGuards(BTC, Guards), SE, TTI))
+ if (shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, SE, TTI))
DesiredPeelCountLast = 1;
- else
- assert(!shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, BTC, SE,
- TTI) &&
- "loop guards pessimized result");
return;
}
More information about the llvm-commits
mailing list