[llvm] [VPlan] Fix sentinel assertion when broadcasting invoke results (PR #210464)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 24 03:02:22 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/210464
>From 48170e3ebd4b5c343a8c51f546f083fd1184d2d9 Mon Sep 17 00:00:00 2001
From: Kristianerik <kristian.erik at outlook.com>
Date: Fri, 17 Jul 2026 16:46:41 -0700
Subject: [PATCH 1/2] [VPlan] Use getInsertionPointAfterDef to fix sentinel
assertion on invoke
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 11 +--
.../LoopVectorize/X86/invoke-in-preheader.ll | 99 +++++++++++++++++++
2 files changed, 104 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 7d69b3453cba9..7c9d528804049 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -337,12 +337,11 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
VPLane LastLane = VPLane::getLastLaneForVF(VF);
IRBuilderBase::InsertPointGuard Guard(Builder);
if (auto *LastInst = dyn_cast<Instruction>(get(Def, LastLane)))
- // Set the insert point after the last scalarized instruction or after the
- // last PHI, if LastInst is a PHI. This ensures the insertelement sequence
- // will directly follow the scalar definitions.
- Builder.SetInsertPoint(isa<PHINode>(LastInst)
- ? LastInst->getParent()->getFirstNonPHIIt()
- : std::next(BasicBlock::iterator(LastInst)));
+ // Set the insert point after the last scalarized instruction. This
+ // ensures the insertelement sequence will directly follow the scalar
+ // definitions.
+ if (auto InsertPt = LastInst->getInsertionPointAfterDef())
+ Builder.SetInsertPoint(*InsertPt);
Value *VectorValue = GetBroadcastInstrs(ScalarValue);
set(Def, VectorValue);
return VectorValue;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll b/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
new file mode 100644
index 0000000000000..1ffe8ecb617d2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -S %s | FileCheck %s
+; See https://github.com/llvm/llvm-project/issues/210342
+declare i32 @__gxx_personality_v0(...)
+declare i32 @foo(i32)
+
+define i32 @test_invoke_broadcast() #0 gc "statepoint-example" personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define i32 @test_invoke_broadcast(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] gc "statepoint-example" personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[INVOKE:%.*]] = invoke i32 @foo(i32 0)
+; CHECK-NEXT: to label %[[ITER_CHECK:.*]] unwind label %[[UNWIND:.*]]
+; CHECK: [[ITER_CHECK]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[INVOKE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[INVOKE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = mul <8 x i32> [[BROADCAST_SPLAT2]], [[TMP0]]
+; CHECK-NEXT: [[BIN_RDX3:%.*]] = mul <8 x i32> [[BROADCAST_SPLAT2]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX4:%.*]] = mul <8 x i32> [[BROADCAST_SPLAT2]], [[BIN_RDX3]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[BIN_RDX4]])
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF0:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> splat (i32 1), i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[INVOKE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP2]], %[[VEC_EPILOG_PH]] ], [ [[TMP3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3]] = mul <4 x i32> [[VEC_PHI]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 36
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i64 [ 144, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 128, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi i32 [ [[TMP5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[UNWIND]]:
+; CHECK-NEXT: [[LPAD:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: ret i32 0
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL7]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ [[MUL:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[MUL]] = mul i32 [[ACC]], [[INVOKE]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[IV]], 139
+; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[MUL]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+entry:
+ %invoke = invoke i32 @foo(i32 0)
+ to label %loop unwind label %unwind
+
+unwind:
+ %lpad = landingpad { ptr, i32 }
+ cleanup
+ ret i32 0
+
+loop:
+ %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+ %acc = phi i32 [ %mul, %loop ], [ 0, %entry ]
+ %mul = mul i32 %acc, %invoke
+ %iv.next = add i64 %iv, 4
+ %cmp = icmp ugt i64 %iv, 139
+ br i1 %cmp, label %exit, label %loop
+
+exit:
+ %result = phi i32 [ %mul, %loop ]
+ ret i32 %result
+}
+
+attributes #0 = { "target-cpu"="skylake-avx512" }
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 4, i32 28}
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META2]]}
+;.
>From 5977826b294b1620502a4fd9a48b9d7d845b1906 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 24 Jul 2026 11:02:11 +0100
Subject: [PATCH 2/2] Update invoke-in-preheader.ll
---
.../LoopVectorize/X86/invoke-in-preheader.ll | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll b/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
index 1ffe8ecb617d2..c4c60fa08e572 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
@@ -1,10 +1,11 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -S %s | FileCheck %s
-; See https://github.com/llvm/llvm-project/issues/210342
+
declare i32 @__gxx_personality_v0(...)
declare i32 @foo(i32)
-define i32 @test_invoke_broadcast() #0 gc "statepoint-example" personality ptr @__gxx_personality_v0 {
+; See https://github.com/llvm/llvm-project/issues/210342
+define i32 @test_invoke_broadcast() #0 personality ptr @__gxx_personality_v0 {
; CHECK-LABEL: define i32 @test_invoke_broadcast(
; CHECK-SAME: ) #[[ATTR0:[0-9]+]] gc "statepoint-example" personality ptr @__gxx_personality_v0 {
; CHECK-NEXT: [[ENTRY:.*:]]
@@ -90,10 +91,3 @@ exit:
}
attributes #0 = { "target-cpu"="skylake-avx512" }
-;.
-; CHECK: [[PROF0]] = !{!"branch_weights", i32 4, i32 28}
-; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
-; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META2]]}
-;.
More information about the llvm-commits
mailing list