[llvm] 57fd7ff - [IndVarSimplify] Lift limitations on IV being a Phi for turn-to-invariant
Max Kazantsev via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 21 22:09:55 PST 2022
Author: Max Kazantsev
Date: 2022-11-22T12:53:37+07:00
New Revision: 57fd7ffefffae313de800fecdd9f095a17bfd4ea
URL: https://github.com/llvm/llvm-project/commit/57fd7ffefffae313de800fecdd9f095a17bfd4ea
DIFF: https://github.com/llvm/llvm-project/commit/57fd7ffefffae313de800fecdd9f095a17bfd4ea.diff
LOG: [IndVarSimplify] Lift limitations on IV being a Phi for turn-to-invariant
These limitations are too strict, and their only purpose is to avoid code
size explosion. These restrictions seem obsolete, and the size problem
is solved in other places through cheap expansion limits.
The motivation is that the old code cannot deal with comparisons against
induction variant's increment.
Differential Revision: https://reviews.llvm.org/D138412
Reviewed By: lebedev.ri, reames
Added:
Modified:
llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll
llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 9e23fa9af786b..db7a6d5c63912 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -195,6 +195,9 @@ Value *SimplifyIndvar::foldIVUser(Instruction *UseInst, Instruction *IVOperand)
bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp,
Instruction *IVOperand) {
+ auto *Preheader = L->getLoopPreheader();
+ if (!Preheader)
+ return false;
unsigned IVOperIdx = 0;
ICmpInst::Predicate Pred = ICmp->getPredicate();
if (IVOperand != ICmp->getOperand(0)) {
@@ -209,11 +212,6 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp,
const Loop *ICmpLoop = LI->getLoopFor(ICmp->getParent());
const SCEV *S = SE->getSCEVAtScope(ICmp->getOperand(IVOperIdx), ICmpLoop);
const SCEV *X = SE->getSCEVAtScope(ICmp->getOperand(1 - IVOperIdx), ICmpLoop);
-
- auto *PN = dyn_cast<PHINode>(IVOperand);
- if (!PN)
- return false;
-
auto LIP = SE->getLoopInvariantPredicate(Pred, S, X, L, ICmp);
if (!LIP)
return false;
@@ -221,40 +219,18 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp,
const SCEV *InvariantLHS = LIP->LHS;
const SCEV *InvariantRHS = LIP->RHS;
- // Rewrite the comparison to a loop invariant comparison if it can be done
- // cheaply, where cheaply means "we don't need to emit any new
- // instructions".
-
- SmallDenseMap<const SCEV*, Value*> CheapExpansions;
- CheapExpansions[S] = ICmp->getOperand(IVOperIdx);
- CheapExpansions[X] = ICmp->getOperand(1 - IVOperIdx);
-
- // TODO: Support multiple entry loops? (We currently bail out of these in
- // the IndVarSimplify pass)
- if (auto *BB = L->getLoopPredecessor()) {
- const int Idx = PN->getBasicBlockIndex(BB);
- if (Idx >= 0) {
- Value *Incoming = PN->getIncomingValue(Idx);
- const SCEV *IncomingS = SE->getSCEV(Incoming);
- CheapExpansions[IncomingS] = Incoming;
- }
- }
- Value *NewLHS = CheapExpansions[InvariantLHS];
- Value *NewRHS = CheapExpansions[InvariantRHS];
-
- if (!NewLHS)
- if (auto *ConstLHS = dyn_cast<SCEVConstant>(InvariantLHS))
- NewLHS = ConstLHS->getValue();
- if (!NewRHS)
- if (auto *ConstRHS = dyn_cast<SCEVConstant>(InvariantRHS))
- NewRHS = ConstRHS->getValue();
-
- if (!NewLHS || !NewRHS)
- // We could not find an existing value to replace either LHS or RHS.
- // Generating new instructions has subtler tradeoffs, so avoid doing that
- // for now.
+ // Do not generate something ridiculous.
+ auto *PHTerm = Preheader->getTerminator();
+ if (Rewriter.isHighCostExpansion(InvariantLHS, L, SCEVCheapExpansionBudget,
+ TTI, PHTerm))
return false;
-
+ if (Rewriter.isHighCostExpansion(InvariantRHS, L, SCEVCheapExpansionBudget,
+ TTI, PHTerm))
+ return false;
+ auto *NewLHS =
+ Rewriter.expandCodeFor(InvariantLHS, IVOperand->getType(), PHTerm);
+ auto *NewRHS =
+ Rewriter.expandCodeFor(InvariantRHS, IVOperand->getType(), PHTerm);
LLVM_DEBUG(dbgs() << "INDVARS: Simplified comparison: " << *ICmp << '\n');
ICmp->setPredicate(InvariantPredicate);
ICmp->setOperand(0, NewLHS);
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll b/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll
index 5ce4539e4bf29..a27ea770d1159 100644
--- a/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll
+++ b/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll
@@ -29,11 +29,10 @@ for.end: ; preds = %if.end, %entry
define void @test1.next(i64 %start) {
; CHECK-LABEL: @test1.next(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END:%.*]], label [[LOOP]]
; CHECK: for.end:
; CHECK-NEXT: ret void
@@ -77,11 +76,10 @@ for.end: ; preds = %if.end, %entry
define void @test2.next(i64 %start) {
; CHECK-LABEL: @test2.next(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[CMP1:%.*]] = icmp sle i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: [[CMP1:%.*]] = icmp sle i64 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END:%.*]], label [[LOOP]]
; CHECK: for.end:
; CHECK-NEXT: ret void
@@ -138,15 +136,16 @@ for.end: ; preds = %if.end, %entry
define void @test3.next(i64 %start) {
; CHECK-LABEL: @test3.next(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25
; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]]
; CHECK: backedge:
; CHECK-NEXT: call void @foo()
-; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END]], label [[LOOP]]
; CHECK: for.end:
; CHECK-NEXT: ret void
@@ -209,15 +208,16 @@ for.end: ; preds = %if.end, %entry
define void @test4.next(i64 %start) {
; CHECK-LABEL: @test4.next(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25
; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]]
; CHECK: backedge:
; CHECK-NEXT: call void @foo()
-; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i64 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP]], label [[FOR_END]]
; CHECK: for.end:
; CHECK-NEXT: ret void
@@ -279,15 +279,16 @@ for.end: ; preds = %if.end, %entry
define void @test5.next(i64 %start) {
; CHECK-LABEL: @test5.next(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nuw i64 [[START:%.*]], 1
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25
; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]]
; CHECK: backedge:
; CHECK-NEXT: call void @foo()
-; CHECK-NEXT: [[CMP1:%.*]] = icmp ugt i64 [[INDVARS_IV_NEXT]], 101
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ugt i64 [[TMP0]], 101
; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP]], label [[FOR_END]]
; CHECK: for.end:
; CHECK-NEXT: ret void
@@ -350,15 +351,16 @@ for.end: ; preds = %if.end, %entry
define void @test6.next(i64 %start) {
; CHECK-LABEL: @test6.next(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add nuw i64 [[START:%.*]], 1
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25
; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]]
; CHECK: backedge:
; CHECK-NEXT: call void @foo()
-; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 101
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i64 [[TMP0]], 101
; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END]], label [[LOOP]]
; CHECK: for.end:
; CHECK-NEXT: ret void
@@ -420,11 +422,10 @@ define void @test7.next(i64 %start, i64* %inc_ptr) {
; CHECK-NEXT: [[OK:%.*]] = icmp sge i64 [[INC]], 0
; CHECK-NEXT: br i1 [[OK]], label [[LOOP_PREHEADER:%.*]], label [[FOR_END:%.*]]
; CHECK: loop.preheader:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INC]], [[START:%.*]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], [[INC]]
-; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 0
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[TMP0]], 0
; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END_LOOPEXIT:%.*]], label [[LOOP]]
; CHECK: for.end.loopexit:
; CHECK-NEXT: br label [[FOR_END]]
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll
index 08633344276c1..8dbb2a9126855 100644
--- a/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll
+++ b/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll
@@ -8,15 +8,16 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test(i32 %start) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START:%.*]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START:%.*]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[START]] to i64
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: backedge:
; CHECK-NEXT: br label [[LOOP]]
; CHECK: loop:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ [[TMP1]], [[ENTRY:%.*]] ]
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
; CHECK-NEXT: [[INDVARS:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; CHECK-NEXT: [[LOOP_EXIT_COND:%.*]] = icmp slt i32 [[INDVARS]], 11
+; CHECK-NEXT: [[LOOP_EXIT_COND:%.*]] = icmp slt i32 [[TMP0]], 11
; CHECK-NEXT: br i1 [[LOOP_EXIT_COND]], label [[EXIT:%.*]], label [[STUCK_PREHEADER:%.*]]
; CHECK: stuck.preheader:
; CHECK-NEXT: br label [[STUCK:%.*]]
More information about the llvm-commits
mailing list