[PATCH] D121618: [LV] Move code to place pointer induction increment to VPlan post-processing.
Florian Hahn via Phabricator via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 28 08:42:59 PDT 2022
fhahn updated this revision to Diff 418600.
fhahn marked 4 inline comments as done.
fhahn added a comment.
Address latest comments, thanks!
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D121618/new/
https://reviews.llvm.org/D121618
Files:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
Index: llvm/lib/Transforms/Vectorize/VPlan.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -981,14 +981,25 @@
}
for (VPRecipeBase &R : Header->phis()) {
// Skip phi-like recipes that generate their backedege values themselves.
- // TODO: Model their backedge values explicitly.
- if (isa<VPWidenPHIRecipe>(&R) || isa<VPWidenPointerInductionRecipe>(&R))
+ if (isa<VPWidenPHIRecipe>(&R))
continue;
- // Set the correct incoming block for backedge values and move induction to
- // latch.
- if (auto *IndR = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
- auto *Phi = cast<PHINode>(State->get(IndR, 0));
+ if (isa<VPWidenPointerInductionRecipe>(&R) ||
+ isa<VPWidenIntOrFpInductionRecipe>(&R)) {
+ PHINode *Phi = nullptr;
+ if (auto *WidenPhi = dyn_cast<VPWidenPointerInductionRecipe>(&R)) {
+ // TODO: Split off the case that all users of a pointer phi are scalar
+ // from the VPWidenPointerInductionRecipe.
+ if (all_of(WidenPhi->users(), [WidenPhi](const VPUser *U) {
+ return cast<VPRecipeBase>(U)->usesScalars(WidenPhi);
+ }))
+ continue;
+
+ auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi, 0));
+ Phi = cast<PHINode>(GEP->getPointerOperand());
+ } else
+ Phi = cast<PHINode>(State->get(R.getVPSingleValue(), 0));
+
Phi->setIncomingBlock(1, VectorLatchBB);
// Move the last step to the end of the latch block. This ensures
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9599,11 +9599,9 @@
NewPointerPhi->addIncoming(ScalarStartValue, State.CFG.VectorPreHeader);
// A pointer induction, performed by using a gep
- BasicBlock *LoopLatch =
- State.LI->getLoopFor(State.CFG.PrevBB)->getLoopLatch();
+ const DataLayout &DL = CanonicalIV->getModule()->getDataLayout();
+ Instruction *InductionLoc = &*State.Builder.GetInsertPoint();
- const DataLayout &DL = LoopLatch->getModule()->getDataLayout();
- Instruction *InductionLoc = LoopLatch->getTerminator();
const SCEV *ScalarStep = IndDesc.getStep();
SCEVExpander Exp(SE, DL, "induction");
Value *ScalarStepValue = Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc);
@@ -9614,7 +9612,12 @@
IndDesc.getElementType(), NewPointerPhi,
State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
InductionLoc);
- NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
+ // Add induction update using an incorrect block temporarily. The phi node
+ // will be fixed after VPlan execution. Note that at this point the latch
+ // block cannot be used, as it does not exist yet.
+ // TODO: Model increment value in VPlan, by turning the recipe into a
+ // multi-def and a subclass of VPHeaderPHIRecipe.
+ NewPointerPhi->addIncoming(InductionGEP, State.CFG.VectorPreHeader);
// Create UF many actual address geps that use the pointer
// phi as base and a vectorized version of the step value
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D121618.418600.patch
Type: text/x-patch
Size: 3272 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20220328/302f6aa5/attachment.bin>
More information about the llvm-commits
mailing list