[PATCH] D121617: [LV] Move code to place induction increment to VPlan post-processing.
Florian Hahn via Phabricator via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 28 08:13:42 PDT 2022
fhahn updated this revision to Diff 418593.
fhahn added a comment.
Thanks Ayal! I rebased the patch and added the TODO. I'll land this one soon.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D121617/new/
https://reviews.llvm.org/D121617
Files:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
Index: llvm/lib/Transforms/Vectorize/VPlan.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -982,10 +982,22 @@
for (VPRecipeBase &R : Header->phis()) {
// Skip phi-like recipes that generate their backedege values themselves.
// TODO: Model their backedge values explicitly.
- if (isa<VPWidenIntOrFpInductionRecipe>(&R) || isa<VPWidenPHIRecipe>(&R) ||
- isa<VPWidenPointerInductionRecipe>(&R))
+ if (isa<VPWidenPHIRecipe>(&R) || isa<VPWidenPointerInductionRecipe>(&R))
continue;
+ // Set the correct incoming block for backedge values and move induction to
+ // latch.
+ if (auto *IndR = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
+ auto *Phi = cast<PHINode>(State->get(IndR, 0));
+ Phi->setIncomingBlock(1, VectorLatchBB);
+
+ // Move the last step to the end of the latch block. This ensures
+ // consistent placement of all induction updates.
+ Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
+ Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+ continue;
+ }
+
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
// For canonical IV, first-order recurrences and in-order reduction phis,
// only a single part is generated, which provides the last part from the
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9534,16 +9534,14 @@
LastInduction->setDebugLoc(EntryVal->getDebugLoc());
}
- // Move the last step to the end of the latch block. This ensures consistent
- // placement of all induction updates.
- auto *LoopVectorLatch =
- State.LI->getLoopFor(State.CFG.PrevBB)->getLoopLatch();
- auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator());
- LastInduction->moveBefore(Br);
LastInduction->setName("vec.ind.next");
-
VecInd->addIncoming(SteppedStart, State.CFG.VectorPreHeader);
- VecInd->addIncoming(LastInduction, LoopVectorLatch);
+ // Add induction update using an incorrect block temporarily. The phi node
+ // will be fixed after VPlan execution. Note that at this point the latch
+ // block cannot be used, as it does not exist yet.
+ // TODO: Model increment value in VPlan, by turning the recipe into a
+ // multi-def and a subclass of VPHeaderPHIRecipe.
+ VecInd->addIncoming(LastInduction, State.CFG.VectorPreHeader);
}
void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D121617.418593.patch
Type: text/x-patch
Size: 2674 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20220328/e0985084/attachment.bin>
More information about the llvm-commits
mailing list