[llvm] e3bfb6a - [VPlan] Make sure recurrence splice is not inserted between phis.

[Florian Hahn via llvm-commits]

Author: Florian Hahn
Date: 2021-11-08T17:42:32Z
New Revision: e3bfb6a14646fd3b344c491a5f46aaebd43090a7

URL: https://github.com/llvm/llvm-project/commit/e3bfb6a14646fd3b344c491a5f46aaebd43090a7
DIFF: https://github.com/llvm/llvm-project/commit/e3bfb6a14646fd3b344c491a5f46aaebd43090a7.diff

LOG: [VPlan] Make sure recurrence splice is not inserted between phis.

All phi-like recipes should be at the beginning of a VPBasicBlock with
no other recipes in between. Ensure that the recurrence-splicing recipe
is not added between phi-like recipes, but after them.

Reviewed By: Ayal

Differential Revision: https://reviews.llvm.org/D111301

Added: 
    

Modified: 
    llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
    llvm/test/Transforms/LoopVectorize/induction.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ec8105aa6122..b7e077c461df 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9489,16 +9489,20 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
     if (!RecurPhi)
       continue;
 
+    VPRecipeBase *PrevRecipe = RecurPhi->getBackedgeRecipe();
+    VPBasicBlock *InsertBlock = PrevRecipe->getParent();
+    auto *Region = GetReplicateRegion(PrevRecipe);
+    if (Region)
+      InsertBlock = cast<VPBasicBlock>(Region->getSingleSuccessor());
+    if (Region || PrevRecipe->isPhi())
+      Builder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi());
+    else
+      Builder.setInsertPoint(InsertBlock, std::next(PrevRecipe->getIterator()));
+
     auto *RecurSplice = cast<VPInstruction>(
         Builder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
                              {RecurPhi, RecurPhi->getBackedgeValue()}));
 
-    VPRecipeBase *PrevRecipe = RecurPhi->getBackedgeRecipe();
-    if (auto *Region = GetReplicateRegion(PrevRecipe)) {
-      VPBasicBlock *Succ = cast<VPBasicBlock>(Region->getSingleSuccessor());
-      RecurSplice->moveBefore(*Succ, Succ->getFirstNonPhi());
-    } else
-      RecurSplice->moveAfter(PrevRecipe);
     RecurPhi->replaceAllUsesWith(RecurSplice);
     // Set the first operand of RecurSplice to RecurPhi again, after replacing
     // all users.

diff  --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 6de7b0737347..b14ac068c759 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -851,8 +851,10 @@ for.end:
 }
 
 ; Ensure that the shuffle vector for first order recurrence is inserted
-; correctly after all the phis. These new phis correspond to new IVs 
-; that are generated by optimizing non-free truncs of IVs to IVs themselves 
+; correctly after all the phis. These new phis correspond to new IVs
+; that are generated by optimizing non-free truncs of IVs to IVs themselves.
+; This also ensures the first-order recurrence splice recipe is placed
+; correctly if it is fed by an induction.
 define i64 @trunc_with_first_order_recurrence() {
 ; CHECK-LABEL: trunc_with_first_order_recurrence
 ; CHECK-LABEL: vector.body: