[llvm] [SimplifyIndVar] Push more users to worklist for simplifyUsers (PR #93598)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 26 15:27:57 PDT 2024
https://github.com/v01dXYZ updated https://github.com/llvm/llvm-project/pull/93598
>From d1a22f3b0f8979e8dbdf4b177685be16f6ffb8d2 Mon Sep 17 00:00:00 2001
From: v01dxyz <v01dxyz at v01d.xyz>
Date: Thu, 27 Jun 2024 00:15:46 +0200
Subject: [PATCH 1/4] [SimplifyIndVar] Push more users to worklist for
simplifyUsers
Instead of considering only users that are inside the loop,
consider all the users with parent dominated by the loop header.
---
.../llvm/Transforms/Utils/SimplifyIndVar.h | 11 ++-
llvm/lib/Transforms/Scalar/IndVarSimplify.cpp | 11 ++-
llvm/lib/Transforms/Utils/SimplifyIndVar.cpp | 76 +++++++++++--------
3 files changed, 60 insertions(+), 38 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
index bd1718dd8cad7..dbf2bdd18467d 100644
--- a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
+++ b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
@@ -52,12 +52,11 @@ class IVVisitor {
/// where the first entry indicates that the function makes changes and the
/// second entry indicates that it introduced new opportunities for loop
/// unswitching.
-std::pair<bool, bool> simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE,
- DominatorTree *DT, LoopInfo *LI,
- const TargetTransformInfo *TTI,
- SmallVectorImpl<WeakTrackingVH> &Dead,
- SCEVExpander &Rewriter,
- IVVisitor *V = nullptr);
+std::pair<bool, bool>
+simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, DominatorTree *DT,
+ LoopInfo *LI, const TargetTransformInfo *TTI,
+ SmallVectorImpl<WeakTrackingVH> &Dead, SCEVExpander &Rewriter,
+ unsigned MaxDepthOutOfLoop = 1, IVVisitor *V = nullptr);
/// SimplifyLoopIVs - Simplify users of induction variables within this
/// loop. This does not actually change or add IVs.
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index dd7c89034ca09..ea0f7a986e4b9 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -124,6 +124,12 @@ static cl::opt<bool>
AllowIVWidening("indvars-widen-indvars", cl::Hidden, cl::init(true),
cl::desc("Allow widening of indvars to eliminate s/zext"));
+static cl::opt<unsigned> MaxDepthOutOfLoop(
+ "indvars-max-depth-out-of-loop", cl::Hidden, cl::init(1),
+ cl::desc(
+ "Strict upper bound for the number of successive out-of-loop blocks "
+ "when traversing use-def chains. 0 enables full traversal"));
+
namespace {
class IndVarSimplify {
@@ -624,8 +630,9 @@ bool IndVarSimplify::simplifyAndExtend(Loop *L,
// Information about sign/zero extensions of CurrIV.
IndVarSimplifyVisitor Visitor(CurrIV, SE, TTI, DT);
- const auto &[C, U] = simplifyUsersOfIV(CurrIV, SE, DT, LI, TTI, DeadInsts,
- Rewriter, &Visitor);
+ const auto &[C, U] =
+ simplifyUsersOfIV(CurrIV, SE, DT, LI, TTI, DeadInsts, Rewriter,
+ MaxDepthOutOfLoop, &Visitor);
Changed |= C;
RunUnswitching |= U;
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 74af0ef40fe7f..a3c1134cd7d64 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -62,13 +62,18 @@ namespace {
bool Changed = false;
bool RunUnswitching = false;
+ // When following the def-use chains, it can go outside the loop.
+ // Strict upper bound on number of traversed out-of-loop blocks.
+ unsigned MaxDepthOutOfLoop;
+
public:
SimplifyIndvar(Loop *Loop, ScalarEvolution *SE, DominatorTree *DT,
LoopInfo *LI, const TargetTransformInfo *TTI,
SCEVExpander &Rewriter,
- SmallVectorImpl<WeakTrackingVH> &Dead)
+ SmallVectorImpl<WeakTrackingVH> &Dead,
+ unsigned MaxDepthOutOfLoop = 1)
: L(Loop), LI(LI), SE(SE), DT(DT), TTI(TTI), Rewriter(Rewriter),
- DeadInsts(Dead) {
+ DeadInsts(Dead), MaxDepthOutOfLoop(MaxDepthOutOfLoop) {
assert(LI && "IV simplification requires LoopInfo");
}
@@ -80,10 +85,11 @@ namespace {
/// all simplifications to users of an IV.
void simplifyUsers(PHINode *CurrIV, IVVisitor *V = nullptr);
- void pushIVUsers(Instruction *Def,
- SmallPtrSet<Instruction *, 16> &Simplified,
- SmallVectorImpl<std::pair<Instruction *, Instruction *>>
- &SimpleIVUsers);
+ void pushIVUsers(
+ Instruction *Def, SmallPtrSet<Instruction *, 16> &Simplified,
+ SmallVectorImpl<std::tuple<Instruction *, Instruction *, unsigned>>
+ &SimpleIVUsers,
+ unsigned OutOfLoopChainCounter);
Value *foldIVUser(Instruction *UseInst, Instruction *IVOperand);
@@ -514,8 +520,8 @@ bool SimplifyIndvar::eliminateTrunc(TruncInst *TI) {
!DT->isReachableFromEntry(cast<Instruction>(U)->getParent()))
continue;
ICmpInst *ICI = dyn_cast<ICmpInst>(U);
- if (!ICI) return false;
- assert(L->contains(ICI->getParent()) && "LCSSA form broken?");
+ if (!ICI)
+ return false;
if (!(ICI->getOperand(0) == TI && L->isLoopInvariant(ICI->getOperand(1))) &&
!(ICI->getOperand(1) == TI && L->isLoopInvariant(ICI->getOperand(0))))
return false;
@@ -846,7 +852,9 @@ bool SimplifyIndvar::strengthenRightShift(BinaryOperator *BO,
/// Add all uses of Def to the current IV's worklist.
void SimplifyIndvar::pushIVUsers(
Instruction *Def, SmallPtrSet<Instruction *, 16> &Simplified,
- SmallVectorImpl<std::pair<Instruction *, Instruction *>> &SimpleIVUsers) {
+ SmallVectorImpl<std::tuple<Instruction *, Instruction *, unsigned>>
+ &SimpleIVUsers,
+ unsigned OutOfLoopChainCounter) {
for (User *U : Def->users()) {
Instruction *UI = cast<Instruction>(U);
@@ -857,16 +865,23 @@ void SimplifyIndvar::pushIVUsers(
if (UI == Def)
continue;
- // Only change the current Loop, do not change the other parts (e.g. other
- // Loops).
- if (!L->contains(UI))
+ // Avoid adding Defs that SCEV expand to themselves, e.g. the LoopPhis
+ // of the outter loops.
+ if (!DT->dominates(L->getHeader(), UI->getParent()))
continue;
// Do not push the same instruction more than once.
if (!Simplified.insert(UI).second)
continue;
- SimpleIVUsers.push_back(std::make_pair(UI, Def));
+ unsigned Counter =
+ L->contains(UI)
+ ? 0 // reset depth if we go back inside the loop.
+ : OutOfLoopChainCounter + (UI->getParent() != Def->getParent());
+
+ if (!MaxDepthOutOfLoop || Counter < MaxDepthOutOfLoop) {
+ SimpleIVUsers.push_back(std::make_tuple(UI, Def, Counter));
+ }
}
}
@@ -911,17 +926,17 @@ void SimplifyIndvar::simplifyUsers(PHINode *CurrIV, IVVisitor *V) {
SmallPtrSet<Instruction*,16> Simplified;
// Use-def pairs if IV users waiting to be processed for CurrIV.
- SmallVector<std::pair<Instruction*, Instruction*>, 8> SimpleIVUsers;
+ SmallVector<std::tuple<Instruction *, Instruction *, unsigned>, 8>
+ SimpleIVUsers;
// Push users of the current LoopPhi. In rare cases, pushIVUsers may be
// called multiple times for the same LoopPhi. This is the proper thing to
// do for loop header phis that use each other.
- pushIVUsers(CurrIV, Simplified, SimpleIVUsers);
+ pushIVUsers(CurrIV, Simplified, SimpleIVUsers, 0);
while (!SimpleIVUsers.empty()) {
- std::pair<Instruction*, Instruction*> UseOper =
- SimpleIVUsers.pop_back_val();
- Instruction *UseInst = UseOper.first;
+ auto [UseInst, IVOperand, OutOfLoopChainCounter] =
+ SimpleIVUsers.pop_back_val();
// If a user of the IndVar is trivially dead, we prefer just to mark it dead
// rather than try to do some complex analysis or transformation (such as
@@ -945,11 +960,11 @@ void SimplifyIndvar::simplifyUsers(PHINode *CurrIV, IVVisitor *V) {
if ((isa<PtrToIntInst>(UseInst)) || (isa<TruncInst>(UseInst)))
for (Use &U : UseInst->uses()) {
Instruction *User = cast<Instruction>(U.getUser());
- if (replaceIVUserWithLoopInvariant(User))
+ if (DT->dominates(L->getHeader(), User->getParent()) &&
+ replaceIVUserWithLoopInvariant(User))
break; // done replacing
}
- Instruction *IVOperand = UseOper.second;
for (unsigned N = 0; IVOperand; ++N) {
assert(N <= Simplified.size() && "runaway iteration");
(void) N;
@@ -963,7 +978,7 @@ void SimplifyIndvar::simplifyUsers(PHINode *CurrIV, IVVisitor *V) {
continue;
if (eliminateIVUser(UseInst, IVOperand)) {
- pushIVUsers(IVOperand, Simplified, SimpleIVUsers);
+ pushIVUsers(IVOperand, Simplified, SimpleIVUsers, OutOfLoopChainCounter);
continue;
}
@@ -971,14 +986,15 @@ void SimplifyIndvar::simplifyUsers(PHINode *CurrIV, IVVisitor *V) {
if (strengthenBinaryOp(BO, IVOperand)) {
// re-queue uses of the now modified binary operator and fall
// through to the checks that remain.
- pushIVUsers(IVOperand, Simplified, SimpleIVUsers);
+ pushIVUsers(IVOperand, Simplified, SimpleIVUsers,
+ OutOfLoopChainCounter);
}
}
// Try to use integer induction for FPToSI of float induction directly.
if (replaceFloatIVWithIntegerIV(UseInst)) {
// Re-queue the potentially new direct uses of IVOperand.
- pushIVUsers(IVOperand, Simplified, SimpleIVUsers);
+ pushIVUsers(IVOperand, Simplified, SimpleIVUsers, OutOfLoopChainCounter);
continue;
}
@@ -988,7 +1004,7 @@ void SimplifyIndvar::simplifyUsers(PHINode *CurrIV, IVVisitor *V) {
continue;
}
if (isSimpleIVUser(UseInst, L, SE)) {
- pushIVUsers(UseInst, Simplified, SimpleIVUsers);
+ pushIVUsers(UseInst, Simplified, SimpleIVUsers, OutOfLoopChainCounter);
}
}
}
@@ -1002,13 +1018,13 @@ void IVVisitor::anchor() { }
/// Returns a pair where the first entry indicates that the function makes
/// changes and the second entry indicates that it introduced new opportunities
/// for loop unswitching.
-std::pair<bool, bool> simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE,
- DominatorTree *DT, LoopInfo *LI,
- const TargetTransformInfo *TTI,
- SmallVectorImpl<WeakTrackingVH> &Dead,
- SCEVExpander &Rewriter, IVVisitor *V) {
+std::pair<bool, bool>
+simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, DominatorTree *DT,
+ LoopInfo *LI, const TargetTransformInfo *TTI,
+ SmallVectorImpl<WeakTrackingVH> &Dead, SCEVExpander &Rewriter,
+ unsigned MaxDepthOutOfLoop, IVVisitor *V) {
SimplifyIndvar SIV(LI->getLoopFor(CurrIV->getParent()), SE, DT, LI, TTI,
- Rewriter, Dead);
+ Rewriter, Dead, MaxDepthOutOfLoop);
SIV.simplifyUsers(CurrIV, V);
return {SIV.hasChanged(), SIV.runUnswitching()};
}
>From e0d9d0083efe7efe03b95d4c890087f9217caa57 Mon Sep 17 00:00:00 2001
From: v01dxyz <v01dxyz at v01d.xyz>
Date: Wed, 26 Jun 2024 23:38:48 +0200
Subject: [PATCH 2/4] Add simple tests
---
.../simplify-indvars-post-loop.ll | 112 ++++++++++++++++++
1 file changed, 112 insertions(+)
create mode 100644 llvm/test/Transforms/IndVarSimplify/simplify-indvars-post-loop.ll
diff --git a/llvm/test/Transforms/IndVarSimplify/simplify-indvars-post-loop.ll b/llvm/test/Transforms/IndVarSimplify/simplify-indvars-post-loop.ll
new file mode 100644
index 0000000000000..20da68f946345
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/simplify-indvars-post-loop.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -S -passes=indvars --indvars-max-depth-out-of-loop=0 | FileCheck %s
+
+define dso_local i1 @ugt_add_rec_iv(ptr nocapture noundef readonly %s) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local i1 @ugt_add_rec_iv(
+; CHECK-SAME: ptr nocapture noundef readonly [[S:%.*]]) local_unnamed_addr {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[WHILE_COND:.*]]
+; CHECK: [[WHILE_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp ugt i64 [[I_0]], 1234
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i64 [[I_0]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[TMP0]], -48
+; CHECK-NEXT: [[OR_COND:%.*]] = icmp ult i8 [[TMP1]], 10
+; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[I_0]], 1
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[WHILE_COND]], label %[[WHILE_END]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: ret i1 true
+;
+entry:
+ br label %while.cond
+
+while.cond: ; preds = %while.body, %entry
+ %i.0 = phi i64 [ 0, %entry ], [ %add, %while.body ]
+ %exitcond.not = icmp ugt i64 %i.0, 1234
+ br i1 %exitcond.not, label %while.end, label %while.body
+
+while.body: ; preds = %while.cond
+ %arrayidx = getelementptr inbounds i8, ptr %s, i64 %i.0
+ %0 = load i8, ptr %arrayidx, align 1
+ %1 = add i8 %0, -48
+ %or.cond = icmp ult i8 %1, 10
+ %add = add nuw nsw i64 %i.0, 1
+ br i1 %or.cond, label %while.cond, label %while.end
+
+while.end: ; preds = %while.body, %while.cond
+ %cmp6 = icmp ule i64 %i.0, 1235
+ ret i1 %cmp6
+}
+
+define dso_local i1 @ne_add_rec_iv(ptr nocapture noundef readonly %s) local_unnamed_addr #0 {
+; CHECK-LABEL: define dso_local i1 @ne_add_rec_iv(
+; CHECK-SAME: ptr nocapture noundef readonly [[S:%.*]]) local_unnamed_addr {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[WHILE_COND:.*]]
+; CHECK: [[WHILE_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[I_0]], 1235
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[S]], i64 [[I_0]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[TMP0]], -48
+; CHECK-NEXT: [[OR_COND:%.*]] = icmp ult i8 [[TMP1]], 10
+; CHECK-NEXT: [[ADD]] = add nuw nsw i64 [[I_0]], 1
+; CHECK-NEXT: br i1 [[OR_COND]], label %[[WHILE_COND]], label %[[WHILE_END]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: ret i1 true
+;
+entry:
+ br label %while.cond
+
+while.cond: ; preds = %while.body, %entry
+ %i.0 = phi i64 [ 0, %entry ], [ %add, %while.body ]
+ %exitcond.not = icmp eq i64 %i.0, 1235
+ br i1 %exitcond.not, label %while.end, label %while.body
+
+while.body: ; preds = %while.cond
+ %arrayidx = getelementptr inbounds i8, ptr %s, i64 %i.0
+ %0 = load i8, ptr %arrayidx, align 1
+ %1 = add i8 %0, -48
+ %or.cond = icmp ult i8 %1, 10
+ %add = add nuw nsw i64 %i.0, 1
+ br i1 %or.cond, label %while.cond, label %while.end
+
+while.end: ; preds = %while.body, %while.cond
+ %cmp6 = icmp ule i64 %i.0, 1235
+ ret i1 %cmp6
+}
+
+; sanity check to verify there is no simplification of outer loop phis.
+define dso_local i32 @nested_loop(i32 noundef %0, i32 noundef %1) local_unnamed_addr #0 {
+ %3 = icmp eq i32 %0, 0
+ br i1 %3, label %18, label %4
+
+4: ; preds = %2, %15
+ %5 = phi i32 [ %16, %15 ], [ 0, %2 ]
+ %6 = phi i32 [ %13, %15 ], [ 1, %2 ]
+ %7 = urem i32 %5, %1
+ br label %8
+
+8: ; preds = %4, %8
+ %9 = phi i32 [ %7, %4 ], [ %11, %8 ]
+ %10 = phi i32 [ %6, %4 ], [ %13, %8 ]
+ %11 = add i32 %9, 1
+ %12 = mul i32 %11, %9
+ %13 = add i32 %12, %10
+ %14 = icmp ult i32 %11, %1
+ br i1 %14, label %8, label %15
+
+15: ; preds = %8
+ %16 = add i32 %11, %5
+ %17 = icmp ult i32 %16, %0
+ br i1 %17, label %4, label %18
+
+18: ; preds = %15, %2
+ %19 = phi i32 [ 1, %2 ], [ %13, %15 ]
+ ret i32 %19
+}
>From 3511e43aeb58522b5fde2f9b588f2501f557b63b Mon Sep 17 00:00:00 2001
From: v01dxyz <v01dxyz at v01d.xyz>
Date: Wed, 26 Jun 2024 23:41:19 +0200
Subject: [PATCH 3/4] Default to enable full traversal (it shoud be changed
before merge)
---
llvm/lib/Transforms/Scalar/IndVarSimplify.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index ea0f7a986e4b9..c270841835672 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -125,7 +125,7 @@ AllowIVWidening("indvars-widen-indvars", cl::Hidden, cl::init(true),
cl::desc("Allow widening of indvars to eliminate s/zext"));
static cl::opt<unsigned> MaxDepthOutOfLoop(
- "indvars-max-depth-out-of-loop", cl::Hidden, cl::init(1),
+ "indvars-max-depth-out-of-loop", cl::Hidden, cl::init(0),
cl::desc(
"Strict upper bound for the number of successive out-of-loop blocks "
"when traversing use-def chains. 0 enables full traversal"));
>From aeac9e850e55c3c81942eaf13e05db52b05a3185 Mon Sep 17 00:00:00 2001
From: v01dxyz <v01dxyz at v01d.xyz>
Date: Thu, 27 Jun 2024 00:18:29 +0200
Subject: [PATCH 4/4] typo
---
llvm/lib/Transforms/Utils/SimplifyIndVar.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index a3c1134cd7d64..05b28d91587b3 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -866,7 +866,7 @@ void SimplifyIndvar::pushIVUsers(
continue;
// Avoid adding Defs that SCEV expand to themselves, e.g. the LoopPhis
- // of the outter loops.
+ // of the outer loops.
if (!DT->dominates(L->getHeader(), UI->getParent()))
continue;
@@ -879,9 +879,8 @@ void SimplifyIndvar::pushIVUsers(
? 0 // reset depth if we go back inside the loop.
: OutOfLoopChainCounter + (UI->getParent() != Def->getParent());
- if (!MaxDepthOutOfLoop || Counter < MaxDepthOutOfLoop) {
+ if (!MaxDepthOutOfLoop || Counter < MaxDepthOutOfLoop)
SimpleIVUsers.push_back(std::make_tuple(UI, Def, Counter));
- }
}
}
More information about the llvm-commits
mailing list