[llvm] [VPlan] Canonicalize all inner loop headers and latches in plain CFG. (PR #228763)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 13:13:22 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/228763
The plain CFG mirrors the IR's predecessor and successor order. Canonicalize all loops up front, to avoid verification issues when vectorizing outer loops.
Fixes a crash with the newly added test.
>From 25b584df09e7a4b0bc0b7c7a0dfc1f731cd9b348 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 30 Sep 2026 14:08:03 +0100
Subject: [PATCH] [VPlan] Canonicalize all inner loop headers and latches in
plain CFG.
The plain CFG mirrors the IR's predecessor and successor order.
Canonicalize all loops up front, to avoid verification issues
when vectorizing outer loops.
Fixes a crash with the newly added test.
---
.../Vectorize/VPlanConstruction.cpp | 21 ++++-
.../outer-loop-inner-latch-successors.ll | 89 ++++++++++++++++++-
2 files changed, 105 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 43bd85513ce25..ad9fb0b648497 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -540,7 +540,14 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy,
VPDominatorTree VPDT(Plan);
auto *HeaderVPBB = cast<VPBasicBlock>(Plan.getEntry()->getSingleSuccessor());
- canonicalHeaderAndLatch(HeaderVPBB, VPDT);
+ if (TheLoop->isInnermost()) {
+ canonicalHeaderAndLatch(HeaderVPBB, VPDT);
+ } else {
+ // When vectorizing outer loops, canonicalize all loops up front.
+ SmallVector<VPBlockBase *> Blocks(vp_depth_first_shallow(Plan.getEntry()));
+ for (VPBlockBase *VPB : Blocks)
+ canonicalHeaderAndLatch(VPB, VPDT);
+ }
auto *LatchVPBB = cast<VPBasicBlock>(HeaderVPBB->getPredecessors()[1]);
VPBasicBlock *VecPreheader = Plan.createVPBasicBlock("vector.ph");
@@ -1323,9 +1330,15 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, DebugLoc DL) {
VPDominatorTree VPDT(Plan);
PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
Plan.getEntry());
- for (VPBlockBase *HeaderVPB : POT)
- if (canonicalHeaderAndLatch(HeaderVPB, VPDT))
- createLoopRegion(Plan, HeaderVPB, DL);
+ // Headers and latches have been canonicalized by addInitialSkeleton.
+ for (VPBlockBase *HeaderVPB : POT) {
+ if (!VPBlockUtils::isHeader(HeaderVPB, VPDT))
+ continue;
+ assert(HeaderVPB->getPredecessors()[1]->getSuccessors().back() ==
+ HeaderVPB &&
+ "latch must have the header as last successor");
+ createLoopRegion(Plan, HeaderVPB, DL);
+ }
VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
TopRegion->setName("vector loop");
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
index a238b4f725a49..1e5be39d41e92 100644
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -vplan-verify-each -S %s | FileCheck %s
@A = common global [1024 x i64] zeroinitializer, align 16
@B = common global [1024 x i64] zeroinitializer, align 16
@@ -207,6 +207,93 @@ exit:
ret void
}
+; The inner loop has no dedicated preheader, so LV inserts one, making it the
+; first predecessor of the inner header, while the inner latch has the header as
+; its first successor.
+define void @inner_latch_header_first_successor_inserted_preheader(ptr noalias %dst, i64 %n, i1 %c) {
+; CHECK-LABEL: define void @inner_latch_header_first_successor_inserted_preheader(
+; CHECK-SAME: ptr noalias [[DST:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[OUTER_LATCH5:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[OUTER_LATCH5]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[INNER_PREHEADER1:.*]], label %[[OUTER_LATCH5]]
+; CHECK: [[INNER_PREHEADER1]]:
+; CHECK-NEXT: br label %[[INNER2:.*]]
+; CHECK: [[INNER2]]:
+; CHECK-NEXT: [[IV_INNER3:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER_PREHEADER1]] ], [ [[TMP1:%.*]], %[[INNER2]] ]
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], <4 x i64> [[IV_INNER3]]
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[VEC_IND]], <4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true))
+; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i64> [[IV_INNER3]], splat (i64 1)
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[OUTER_LATCH_LOOPEXIT4:.*]], label %[[INNER2]]
+; CHECK: [[OUTER_LATCH_LOOPEXIT4]]:
+; CHECK-NEXT: br label %[[OUTER_LATCH5]]
+; CHECK: [[OUTER_LATCH5]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
+; CHECK: [[OUTER_HEADER]]:
+; CHECK-NEXT: [[IV_OUTER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: br i1 [[C]], label %[[INNER_PREHEADER:.*]], label %[[OUTER_LATCH]]
+; CHECK: [[INNER_PREHEADER]]:
+; CHECK-NEXT: br label %[[INNER:.*]]
+; CHECK: [[INNER]]:
+; CHECK-NEXT: [[IV_INNER:%.*]] = phi i64 [ [[IV_INNER_NEXT:%.*]], %[[INNER]] ], [ 0, %[[INNER_PREHEADER]] ]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV_INNER]]
+; CHECK-NEXT: store i64 [[IV_OUTER]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
+; CHECK-NEXT: [[INNER_EC:%.*]] = icmp ne i64 [[IV_INNER_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[INNER_EC]], label %[[INNER]], label %[[OUTER_LATCH_LOOPEXIT:.*]]
+; CHECK: [[OUTER_LATCH_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[OUTER_LATCH]]
+; CHECK: [[OUTER_LATCH]]:
+; CHECK-NEXT: [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
+; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.header
+
+outer.header:
+ %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
+ br i1 %c, label %inner, label %outer.latch
+
+inner:
+ %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
+ %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv.inner
+ store i64 %iv.outer, ptr %gep.dst, align 8
+ %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+ %inner.ec = icmp ne i64 %iv.inner.next, %n
+ br i1 %inner.ec, label %inner, label %outer.latch
+
+outer.latch:
+ %iv.outer.next = add nuw nsw i64 %iv.outer, 1
+ %outer.ec = icmp eq i64 %iv.outer.next, %n
+ br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
+
+exit:
+ ret void
+}
+
!1 = distinct !{!1, !2, !3}
!2 = !{!"llvm.loop.vectorize.width", i32 4}
!3 = !{!"llvm.loop.vectorize.enable"}
More information about the llvm-commits
mailing list