[llvm] 0d151f3 - [VPlan] Canonicalize all inner loop headers and latches in plain CFG. (#228763)

via llvm-commits llvm-commits at lists.llvm.org
Sat Oct 3 13:52:22 PDT 2026


Author: Florian Hahn
Date: 2026-10-03T20:52:14Z
New Revision: 0d151f3ebb6af731e641d1474d1992d0dd5a12dd

URL: https://github.com/llvm/llvm-project/commit/0d151f3ebb6af731e641d1474d1992d0dd5a12dd
DIFF: https://github.com/llvm/llvm-project/commit/0d151f3ebb6af731e641d1474d1992d0dd5a12dd.diff

LOG: [VPlan] Canonicalize all inner loop headers and latches in plain CFG. (#228763)

The plain CFG mirrors the IR's predecessor and successor order.
Canonicalize all loops up front, to avoid verification issues when
vectorizing outer loops.

Fixes a crash with the newly added test.

Added: 
    

Modified: 
    llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
    llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 43bd85513ce25..ad9fb0b648497 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -540,7 +540,14 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy,
   VPDominatorTree VPDT(Plan);
 
   auto *HeaderVPBB = cast<VPBasicBlock>(Plan.getEntry()->getSingleSuccessor());
-  canonicalHeaderAndLatch(HeaderVPBB, VPDT);
+  if (TheLoop->isInnermost()) {
+    canonicalHeaderAndLatch(HeaderVPBB, VPDT);
+  } else {
+    // When vectorizing outer loops, canonicalize all loops up front.
+    SmallVector<VPBlockBase *> Blocks(vp_depth_first_shallow(Plan.getEntry()));
+    for (VPBlockBase *VPB : Blocks)
+      canonicalHeaderAndLatch(VPB, VPDT);
+  }
   auto *LatchVPBB = cast<VPBasicBlock>(HeaderVPBB->getPredecessors()[1]);
 
   VPBasicBlock *VecPreheader = Plan.createVPBasicBlock("vector.ph");
@@ -1323,9 +1330,15 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, DebugLoc DL) {
   VPDominatorTree VPDT(Plan);
   PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
       Plan.getEntry());
-  for (VPBlockBase *HeaderVPB : POT)
-    if (canonicalHeaderAndLatch(HeaderVPB, VPDT))
-      createLoopRegion(Plan, HeaderVPB, DL);
+  // Headers and latches have been canonicalized by addInitialSkeleton.
+  for (VPBlockBase *HeaderVPB : POT) {
+    if (!VPBlockUtils::isHeader(HeaderVPB, VPDT))
+      continue;
+    assert(HeaderVPB->getPredecessors()[1]->getSuccessors().back() ==
+               HeaderVPB &&
+           "latch must have the header as last successor");
+    createLoopRegion(Plan, HeaderVPB, DL);
+  }
 
   VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
   TopRegion->setName("vector loop");

diff  --git a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
index a238b4f725a49..1e5be39d41e92 100644
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -vplan-verify-each -S %s | FileCheck %s
 
 @A = common global [1024 x i64] zeroinitializer, align 16
 @B = common global [1024 x i64] zeroinitializer, align 16
@@ -207,6 +207,93 @@ exit:
   ret void
 }
 
+; The inner loop has no dedicated preheader, so LV inserts one, making it the
+; first predecessor of the inner header, while the inner latch has the header as
+; its first successor.
+define void @inner_latch_header_first_successor_inserted_preheader(ptr noalias %dst, i64 %n, i1 %c) {
+; CHECK-LABEL: define void @inner_latch_header_first_successor_inserted_preheader(
+; CHECK-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
+; CHECK-NEXT:    br i1 [[C]], label %[[INNER_PREHEADER1:.*]], label %[[OUTER_LATCH5]]
+; CHECK:       [[INNER_PREHEADER1]]:
+; CHECK-NEXT:    br label %[[INNER2:.*]]
+; CHECK:       [[INNER2]]:
+; CHECK-NEXT:    [[IV_INNER3:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER_PREHEADER1]] ], [ [[TMP1:%.*]], %[[INNER2]] ]
+; CHECK-NEXT:    [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], <4 x i64> [[IV_INNER3]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_IND]], <4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP1]] = add nuw nsw <4 x i64> [[IV_INNER3]], splat (i64 1)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[OUTER_LATCH_LOOPEXIT4:.*]], label %[[INNER2]]
+; CHECK:       [[OUTER_LATCH_LOOPEXIT4]]:
+; CHECK-NEXT:    br label %[[OUTER_LATCH5]]
+; CHECK:       [[OUTER_LATCH5]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[IV_OUTER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    br i1 [[C]], label %[[INNER_PREHEADER:.*]], label %[[OUTER_LATCH]]
+; CHECK:       [[INNER_PREHEADER]]:
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[IV_INNER:%.*]] = phi i64 [ [[IV_INNER_NEXT:%.*]], %[[INNER]] ], [ 0, %[[INNER_PREHEADER]] ]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV_INNER]]
+; CHECK-NEXT:    store i64 [[IV_OUTER]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-NEXT:    [[INNER_EC:%.*]] = icmp ne i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[INNER_EC]], label %[[INNER]], label %[[OUTER_LATCH_LOOPEXIT:.*]]
+; CHECK:       [[OUTER_LATCH_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[OUTER_LATCH]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-NEXT:    [[OUTER_EC:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+  br i1 %c, label %inner, label %outer.latch
+
+inner:
+  %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv.inner
+  store i64 %iv.outer, ptr %gep.dst, align 8
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %inner.ec = icmp ne i64 %iv.inner.next, %n
+  br i1 %inner.ec, label %inner, label %outer.latch
+
+outer.latch:
+  %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+  %outer.ec = icmp eq i64 %iv.outer.next, %n
+  br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+  ret void
+}
+
 !1 = distinct !{!1, !2, !3}
 !2 = !{!"llvm.loop.vectorize.width", i32 4}
 !3 = !{!"llvm.loop.vectorize.enable"}


        


More information about the llvm-commits mailing list