[llvm] 88ce403 - [LV] Add new block to place recurrence splice, if needed.

[Florian Hahn via llvm-commits]

Author: Florian Hahn
Date: 2022-06-21T21:54:37+02:00
New Revision: 88ce403c6aab1d0973d93da1a7a997b722553fe2

URL: https://github.com/llvm/llvm-project/commit/88ce403c6aab1d0973d93da1a7a997b722553fe2
DIFF: https://github.com/llvm/llvm-project/commit/88ce403c6aab1d0973d93da1a7a997b722553fe2.diff

LOG: [LV] Add new block to place recurrence splice, if needed.

In some cases, a recurrence splice instructions needs to be inserted
between to regions, for example if the regions get re-arranged during
sinking.

Fixes #56146.

Added: 
    

Modified: 
    llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
    llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 44a8998f6cb4f..c8015c31b9a21 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8944,7 +8944,11 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
     VPBasicBlock *InsertBlock = PrevRecipe->getParent();
     auto *Region = GetReplicateRegion(PrevRecipe);
     if (Region)
-      InsertBlock = cast<VPBasicBlock>(Region->getSingleSuccessor());
+      InsertBlock = dyn_cast<VPBasicBlock>(Region->getSingleSuccessor());
+    if (!InsertBlock) {
+      InsertBlock = new VPBasicBlock(Region->getName() + ".succ");
+      VPBlockUtils::insertBlockAfter(InsertBlock, Region);
+    }
     if (Region || PrevRecipe->isPhi())
       Builder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi());
     else

diff  --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 08d950d9241e0..07f443dfb325e 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -437,3 +437,85 @@ loop:                                             ; preds = %loop, %entry
 exit:                                             ; preds = %loop
   ret void
 }
+
+define void @need_new_block_after_sinking_pr56146(i32 %x, i32* %src) {
+; CHECK-LABEL: need_new_block_after_sinking_pr56146
+; CHECK:      VPlan 'Initial VPlan for VF={2},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT:   vector.body:
+; CHECK-NEXT:     EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
+; CHECK-NEXT:     FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<%l>
+; CHECK-NEXT:     EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]>
+; CHECK-NEXT:     EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]> vp<[[BTC]]>
+; CHECK-NEXT:   Successor(s): loop.0
+; CHECK-EMPTY:
+; CHECK-NEXT:   loop.0:
+; CHECK-NEXT:   Successor(s): pred.load
+; CHECK-EMPTY:
+; CHECK-NEXT:   <xVFxUF> pred.load: {
+; CHECK-NEXT:     pred.load.entry:
+; CHECK-NEXT:       BRANCH-ON-MASK vp<[[CMP]]>
+; CHECK-NEXT:     Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT:     pred.load.if:
+; CHECK-NEXT:       REPLICATE ir<%l> = load ir<%src> (S->V)
+; CHECK-NEXT:     Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT:     pred.load.continue:
+; CHECK-NEXT:       PHI-PREDICATED-INSTRUCTION vp<[[P_L:%.+]]> = ir<%l>
+; CHECK-NEXT:     No successors
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Successor(s): pred.load.succ
+; CHECK-EMPTY:
+; CHECK-NEXT:   pred.load.succ:
+; CHECK-NEXT:     EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn> ir<%l>
+; CHECK-NEXT:   Successor(s): pred.sdiv
+; CHECK-EMPTY:
+; CHECK-NEXT:   <xVFxUF> pred.sdiv: {
+; CHECK-NEXT:     pred.sdiv.entry:
+; CHECK-NEXT:       BRANCH-ON-MASK vp<[[CMP]]>
+; CHECK-NEXT:     Successor(s): pred.sdiv.if, pred.sdiv.continue
+; CHECK-EMPTY:
+; CHECK-NEXT:     pred.sdiv.if:
+; CHECK-NEXT:       REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x>
+; CHECK-NEXT:     Successor(s): pred.sdiv.continue
+; CHECK-EMPTY:
+; CHECK-NEXT:     pred.sdiv.continue:
+; CHECK-NEXT:       PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val>
+; CHECK-NEXT:     No successors
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Successor(s): loop.1
+; CHECK-EMPTY:
+; CHECK-NEXT:   loop.1:
+; CHECK-NEXT:     EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF +  vp<[[CAN_IV]]>
+; CHECK-NEXT:     EMIT branch-on-count  vp<[[CAN_IV_NEXT]]> vp<[[VEC_TC]]>
+; CHECK-NEXT:   No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+  %.pn = phi i32 [ 0, %entry ], [ %l, %loop ]
+  %val = sdiv i32 %.pn, %x
+  %l = load i32, i32* %src, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp ugt i64 %iv, 3
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}