[llvm] 7969275 - [LV] Use VPValue for SCEV expansion in fixupIVUsers.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu May 4 01:26:20 PDT 2023
Author: Florian Hahn
Date: 2023-05-04T09:25:59+01:00
New Revision: 79692750d250d2e155aaa2b078aeb0451f005cb6
URL: https://github.com/llvm/llvm-project/commit/79692750d250d2e155aaa2b078aeb0451f005cb6
DIFF: https://github.com/llvm/llvm-project/commit/79692750d250d2e155aaa2b078aeb0451f005cb6.diff
LOG: [LV] Use VPValue for SCEV expansion in fixupIVUsers.
The step is already expanded in the VPlan. Use this expansion instead.
This is a step towards modeling fixing up IV users in VPlan.
It also fixes a crash casued by SCEV-expanding the Step expression in
fixupIVUsers, where the IR is in an incomplete state
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D147963
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9c000e114c1cc..4a276bb64486e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -580,7 +580,7 @@ class InnerLoopVectorizer {
void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
Value *VectorTripCount, Value *EndValue,
BasicBlock *MiddleBlock, BasicBlock *VectorHeader,
- VPlan &Plan);
+ VPlan &Plan, VPTransformState &State);
/// Handle all cross-iteration phis in the header.
void fixCrossIterationPHIs(VPTransformState &State);
@@ -3342,7 +3342,8 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
const InductionDescriptor &II,
Value *VectorTripCount, Value *EndValue,
BasicBlock *MiddleBlock,
- BasicBlock *VectorHeader, VPlan &Plan) {
+ BasicBlock *VectorHeader, VPlan &Plan,
+ VPTransformState &State) {
// There are two kinds of external IV usages - those that use the value
// computed in the last iteration (the PHI) and those that use the penultimate
// value (the value that feeds into the phi from the loop latch).
@@ -3370,7 +3371,6 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
auto *UI = cast<Instruction>(U);
if (!OrigLoop->contains(UI)) {
assert(isa<PHINode>(UI) && "Expected LCSSA form");
-
IRBuilder<> B(MiddleBlock->getTerminator());
// Fast-math-flags propagate from the original induction instruction.
@@ -3380,8 +3380,11 @@ void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
Value *CountMinusOne = B.CreateSub(
VectorTripCount, ConstantInt::get(VectorTripCount->getType(), 1));
CountMinusOne->setName("cmo");
- Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(),
- VectorHeader->getTerminator());
+
+ VPValue *StepVPV = Plan.getSCEVExpansion(II.getStep());
+ assert(StepVPV && "step must have been expanded during VPlan execution");
+ Value *Step = StepVPV->isLiveIn() ? StepVPV->getLiveInIRValue()
+ : State.get(StepVPV, 0);
Value *Escape =
emitTransformedIndex(B, CountMinusOne, II.getStartValue(), Step, II);
Escape->setName("ind.escape");
@@ -3740,7 +3743,7 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
fixupIVUsers(Entry.first, Entry.second,
getOrCreateVectorTripCount(VectorLoop->getLoopPreheader()),
IVEndValues[Entry.first], LoopMiddleBlock,
- VectorLoop->getHeader(), Plan);
+ VectorLoop->getHeader(), Plan, State);
}
// Fix LCSSA phis not already fixed earlier. Extracts may need to be generated
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 563421e398d05..d8c41195586ec 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1132,13 +1132,19 @@ bool vputils::onlyFirstLaneUsed(VPValue *Def) {
VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
ScalarEvolution &SE) {
+ if (auto *Expanded = Plan.getSCEVExpansion(Expr))
+ return Expanded;
+ VPValue *Expanded = nullptr;
if (auto *E = dyn_cast<SCEVConstant>(Expr))
- return Plan.getVPValueOrAddLiveIn(E->getValue());
- if (auto *E = dyn_cast<SCEVUnknown>(Expr))
- return Plan.getVPValueOrAddLiveIn(E->getValue());
-
- VPBasicBlock *Preheader = Plan.getEntry();
- VPExpandSCEVRecipe *Step = new VPExpandSCEVRecipe(Expr, SE);
- Preheader->appendRecipe(Step);
- return Step;
+ Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
+ else if (auto *E = dyn_cast<SCEVUnknown>(Expr))
+ Expanded = Plan.getVPValueOrAddLiveIn(E->getValue());
+ else {
+
+ VPBasicBlock *Preheader = Plan.getEntry();
+ Expanded = new VPExpandSCEVRecipe(Expr, SE);
+ Preheader->appendRecipe(Expanded->getDefiningRecipe());
+ }
+ Plan.addSCEVExpansion(Expr, Expanded);
+ return Expanded;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 45fc5041f9e55..770b6c83b22f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2254,6 +2254,11 @@ class VPlan {
/// Values used outside the plan.
MapVector<PHINode *, VPLiveOut *> LiveOuts;
+ /// Mapping from SCEVs to the VPValues representing their expansions.
+ /// NOTE: This mapping is temporary and will be removed once all users have
+ /// been modeled in VPlan directly.
+ DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
+
public:
VPlan(VPBasicBlock *Entry = nullptr) : Entry(Entry) {
if (Entry)
@@ -2417,6 +2422,19 @@ class VPlan {
return LiveOuts;
}
+ VPValue *getSCEVExpansion(const SCEV *S) const {
+ auto I = SCEVToExpansion.find(S);
+ if (I == SCEVToExpansion.end())
+ return nullptr;
+ return I->second;
+ }
+
+ void addSCEVExpansion(const SCEV *S, VPValue *V) {
+ assert(SCEVToExpansion.find(S) == SCEVToExpansion.end() &&
+ "SCEV already expanded");
+ SCEVToExpansion[S] = V;
+ }
+
private:
/// Add to the given dominator tree the header block and every new basic block
/// that was created between it and the latch block, inclusive.
diff --git a/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll
index 49d3da8a2aefd..1216e2d728af2 100644
--- a/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr58811-scev-expansion.ll
@@ -1,9 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
-; REQUIRES: asserts
-; XFAIL: *
-
define void @test1_pr58811() {
; CHECK-LABEL: @test1_pr58811(
; CHECK-NEXT: entry:
More information about the llvm-commits
mailing list