[PATCH] D121618: [LV] Move code to place pointer induction increment to VPlan post-processing.

[Florian Hahn via Phabricator via llvm-commits]

fhahn updated this revision to Diff 418600.
fhahn marked 4 inline comments as done.
fhahn added a comment.

Address latest comments, thanks!


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D121618/new/

https://reviews.llvm.org/D121618

Files:
  llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
  llvm/lib/Transforms/Vectorize/VPlan.cpp


Index: llvm/lib/Transforms/Vectorize/VPlan.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -981,14 +981,25 @@
   }
   for (VPRecipeBase &R : Header->phis()) {
     // Skip phi-like recipes that generate their backedege values themselves.
-    // TODO: Model their backedge values explicitly.
-    if (isa<VPWidenPHIRecipe>(&R) || isa<VPWidenPointerInductionRecipe>(&R))
+    if (isa<VPWidenPHIRecipe>(&R))
       continue;
 
-    // Set the correct incoming block for backedge values and move induction to
-    // latch.
-    if (auto *IndR = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
-      auto *Phi = cast<PHINode>(State->get(IndR, 0));
+    if (isa<VPWidenPointerInductionRecipe>(&R) ||
+        isa<VPWidenIntOrFpInductionRecipe>(&R)) {
+      PHINode *Phi = nullptr;
+      if (auto *WidenPhi = dyn_cast<VPWidenPointerInductionRecipe>(&R)) {
+        // TODO: Split off the case that all users of a pointer phi are scalar
+        // from the VPWidenPointerInductionRecipe.
+        if (all_of(WidenPhi->users(), [WidenPhi](const VPUser *U) {
+              return cast<VPRecipeBase>(U)->usesScalars(WidenPhi);
+            }))
+          continue;
+
+        auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi, 0));
+        Phi = cast<PHINode>(GEP->getPointerOperand());
+      } else
+        Phi = cast<PHINode>(State->get(R.getVPSingleValue(), 0));
+
       Phi->setIncomingBlock(1, VectorLatchBB);
 
       // Move the last step to the end of the latch block. This ensures
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9599,11 +9599,9 @@
   NewPointerPhi->addIncoming(ScalarStartValue, State.CFG.VectorPreHeader);
 
   // A pointer induction, performed by using a gep
-  BasicBlock *LoopLatch =
-      State.LI->getLoopFor(State.CFG.PrevBB)->getLoopLatch();
+  const DataLayout &DL = CanonicalIV->getModule()->getDataLayout();
+  Instruction *InductionLoc = &*State.Builder.GetInsertPoint();
 
-  const DataLayout &DL = LoopLatch->getModule()->getDataLayout();
-  Instruction *InductionLoc = LoopLatch->getTerminator();
   const SCEV *ScalarStep = IndDesc.getStep();
   SCEVExpander Exp(SE, DL, "induction");
   Value *ScalarStepValue = Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc);
@@ -9614,7 +9612,12 @@
       IndDesc.getElementType(), NewPointerPhi,
       State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
       InductionLoc);
-  NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
+  // Add induction update using an incorrect block temporarily. The phi node
+  // will be fixed after VPlan execution. Note that at this point the latch
+  // block cannot be used, as it does not exist yet.
+  // TODO: Model increment value in VPlan, by turning the recipe into a
+  // multi-def and a subclass of VPHeaderPHIRecipe.
+  NewPointerPhi->addIncoming(InductionGEP, State.CFG.VectorPreHeader);
 
   // Create UF many actual address geps that use the pointer
   // phi as base and a vectorized version of the step value


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D121618.418600.patch
Type: text/x-patch
Size: 3272 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20220328/302f6aa5/attachment.bin>