[llvm] 0d151f3 - [VPlan] Canonicalize all inner loop headers and latches in plain CFG. (#228763)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 13:52:22 PDT 2026
Author: Florian Hahn
Date: 2026-10-03T20:52:14Z
New Revision: 0d151f3ebb6af731e641d1474d1992d0dd5a12dd
URL: https://github.com/llvm/llvm-project/commit/0d151f3ebb6af731e641d1474d1992d0dd5a12dd
DIFF: https://github.com/llvm/llvm-project/commit/0d151f3ebb6af731e641d1474d1992d0dd5a12dd.diff
LOG: [VPlan] Canonicalize all inner loop headers and latches in plain CFG. (#228763)
The plain CFG mirrors the IR's predecessor and successor order.
Canonicalize all loops up front, to avoid verification issues when
vectorizing outer loops.
Fixes a crash with the newly added test.
Added:
Modified:
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 43bd85513ce25..ad9fb0b648497 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -540,7 +540,14 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy,
VPDominatorTree VPDT(Plan);
auto *HeaderVPBB = cast<VPBasicBlock>(Plan.getEntry()->getSingleSuccessor());
- canonicalHeaderAndLatch(HeaderVPBB, VPDT);
+ if (TheLoop->isInnermost()) {
+ canonicalHeaderAndLatch(HeaderVPBB, VPDT);
+ } else {
+ // When vectorizing outer loops, canonicalize all loops up front.
+ SmallVector<VPBlockBase *> Blocks(vp_depth_first_shallow(Plan.getEntry()));
+ for (VPBlockBase *VPB : Blocks)
+ canonicalHeaderAndLatch(VPB, VPDT);
+ }
auto *LatchVPBB = cast<VPBasicBlock>(HeaderVPBB->getPredecessors()[1]);
VPBasicBlock *VecPreheader = Plan.createVPBasicBlock("vector.ph");
@@ -1323,9 +1330,15 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, DebugLoc DL) {
VPDominatorTree VPDT(Plan);
PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
Plan.getEntry());
- for (VPBlockBase *HeaderVPB : POT)
- if (canonicalHeaderAndLatch(HeaderVPB, VPDT))
- createLoopRegion(Plan, HeaderVPB, DL);
+ // Headers and latches have been canonicalized by addInitialSkeleton.
+ for (VPBlockBase *HeaderVPB : POT) {
+ if (!VPBlockUtils::isHeader(HeaderVPB, VPDT))
+ continue;
+ assert(HeaderVPB->getPredecessors()[1]->getSuccessors().back() ==
+ HeaderVPB &&
+ "latch must have the header as last successor");
+ createLoopRegion(Plan, HeaderVPB, DL);
+ }
VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
TopRegion->setName("vector loop");
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
index a238b4f725a49..1e5be39d41e92 100644
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -vplan-verify-each -S %s | FileCheck %s
@A = common global [1024 x i64] zeroinitializer, align 16
@B = common global [1024 x i64] zeroinitializer, align 16
@@ -207,6 +207,93 @@ exit:
ret void
}
+; The inner loop has no dedicated preheader, so LV inserts one, making it the
+; first predecessor of the inner header, while the inner latch has the header as
+; its first successor.
+define void @inner_latch_header_first_successor_inserted_preheader(ptr noalias %dst, i64 %n, i1 %c) {
+; CHECK-LABEL: define void @inner_latch_header_first_successor_inserted_preheader(
+; CHECK-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[INNER_PREHEADER1:.*]], label %[[OUTER_LATCH5]]
+; CHECK: [[INNER_PREHEADER1]]:
+; CHECK-NEXT: br label %[[INNER2:.*]]
+; CHECK: [[INNER2]]:
+; CHECK-NEXT: [[IV_INNER3:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER_PREHEADER1]] ], [ [[TMP1:%.*]], %[[INNER2]] ]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], <4 x i64> [[IV_INNER3]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_IND]], <4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i64> [[IV_INNER3]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[OUTER_LATCH_LOOPEXIT4:.*]], label %[[INNER2]]
+; CHECK: [[OUTER_LATCH_LOOPEXIT4]]:
+; CHECK-NEXT: br label %[[OUTER_LATCH5]]
+; CHECK: [[OUTER_LATCH5]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[IV_OUTER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[INNER_PREHEADER:.*]], label %[[OUTER_LATCH]]
+; CHECK: [[INNER_PREHEADER]]:
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[IV_INNER:%.*]] = phi i64 [ [[IV_INNER_NEXT:%.*]], %[[INNER]] ], [ 0, %[[INNER_PREHEADER]] ]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV_INNER]]
+; CHECK-NEXT: store i64 [[IV_OUTER]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-NEXT: [[INNER_EC:%.*]] = icmp ne i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[INNER_EC]], label %[[INNER]], label %[[OUTER_LATCH_LOOPEXIT:.*]]
+; CHECK: [[OUTER_LATCH_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[OUTER_LATCH]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+ br i1 %c, label %inner, label %outer.latch
+
+inner:
+ %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+ %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv.inner
+ store i64 %iv.outer, ptr %gep.dst, align 8
+ %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+ %inner.ec = icmp ne i64 %iv.inner.next, %n
+ br i1 %inner.ec, label %inner, label %outer.latch
+
+outer.latch:
+ %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+ %outer.ec = icmp eq i64 %iv.outer.next, %n
+ br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+ ret void
+}
+
!1 = distinct !{!1, !2, !3}
!2 = !{!"llvm.loop.vectorize.width", i32 4}
!3 = !{!"llvm.loop.vectorize.enable"}
More information about the llvm-commits
mailing list