[llvm-branch-commits] [llvm] release/23.x: [VPlan] Fix sentinel assertion when broadcasting invoke results (#210464) (PR #211781)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 24 05:32:36 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: llvmbot
<details>
<summary>Changes</summary>
Backport 81cedf1e2060dbc2c5be43a6be0c86d6ba76df8d
Requested by: @<!-- -->fhahn
---
Full diff: https://github.com/llvm/llvm-project/pull/211781.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlan.cpp (+5-6)
- (added) llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll (+93)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 7d69b3453cba9..7c9d528804049 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -337,12 +337,11 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
VPLane LastLane = VPLane::getLastLaneForVF(VF);
IRBuilderBase::InsertPointGuard Guard(Builder);
if (auto *LastInst = dyn_cast<Instruction>(get(Def, LastLane)))
- // Set the insert point after the last scalarized instruction or after the
- // last PHI, if LastInst is a PHI. This ensures the insertelement sequence
- // will directly follow the scalar definitions.
- Builder.SetInsertPoint(isa<PHINode>(LastInst)
- ? LastInst->getParent()->getFirstNonPHIIt()
- : std::next(BasicBlock::iterator(LastInst)));
+ // Set the insert point after the last scalarized instruction. This
+ // ensures the insertelement sequence will directly follow the scalar
+ // definitions.
+ if (auto InsertPt = LastInst->getInsertionPointAfterDef())
+ Builder.SetInsertPoint(*InsertPt);
Value *VectorValue = GetBroadcastInstrs(ScalarValue);
set(Def, VectorValue);
return VectorValue;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll b/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
new file mode 100644
index 0000000000000..4baf87a706431
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/invoke-in-preheader.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -S %s | FileCheck %s
+
+declare i32 @__gxx_personality_v0(...)
+declare i32 @foo(i32)
+
+; See https://github.com/llvm/llvm-project/issues/210342
+define i32 @test_invoke_broadcast() #0 personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define i32 @test_invoke_broadcast(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[INVOKE:%.*]] = invoke i32 @foo(i32 0)
+; CHECK-NEXT: to label %[[ITER_CHECK:.*]] unwind label %[[UNWIND:.*]]
+; CHECK: [[ITER_CHECK]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[INVOKE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[INVOKE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = mul <8 x i32> [[BROADCAST_SPLAT2]], [[TMP0]]
+; CHECK-NEXT: [[BIN_RDX3:%.*]] = mul <8 x i32> [[BROADCAST_SPLAT2]], [[BIN_RDX]]
+; CHECK-NEXT: [[BIN_RDX4:%.*]] = mul <8 x i32> [[BROADCAST_SPLAT2]], [[BIN_RDX3]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[BIN_RDX4]])
+; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF0:![0-9]+]]
+; CHECK: [[VEC_EPILOG_PH]]:
+; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 32, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> splat (i32 1), i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> poison, i32 [[INVOKE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP2]], %[[VEC_EPILOG_PH]] ], [ [[TMP3:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP3]] = mul <4 x i32> [[VEC_PHI]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 36
+; CHECK-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i64 [ 144, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 128, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX8:%.*]] = phi i32 [ [[TMP5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP1]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[UNWIND]]:
+; CHECK-NEXT: [[LPAD:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: ret i32 0
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL7]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ [[MUL:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[MUL]] = mul i32 [[ACC]], [[INVOKE]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[IV]], 139
+; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[MUL]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+entry:
+ %invoke = invoke i32 @foo(i32 0)
+ to label %loop unwind label %unwind
+
+unwind:
+ %lpad = landingpad { ptr, i32 }
+ cleanup
+ ret i32 0
+
+loop:
+ %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+ %acc = phi i32 [ %mul, %loop ], [ 0, %entry ]
+ %mul = mul i32 %acc, %invoke
+ %iv.next = add i64 %iv, 4
+ %cmp = icmp ugt i64 %iv, 139
+ br i1 %cmp, label %exit, label %loop
+
+exit:
+ %result = phi i32 [ %mul, %loop ]
+ ret i32 %result
+}
+
+attributes #0 = { "target-cpu"="skylake-avx512" }
``````````
</details>
https://github.com/llvm/llvm-project/pull/211781
More information about the llvm-branch-commits
mailing list