[llvm] [VPlan] Collect FOR PHIs before sinking/hoisting recurrence users (PR #194671)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 08:38:33 PDT 2026
https://github.com/xiongzile updated https://github.com/llvm/llvm-project/pull/194671
>From 55c4c92da9b1632a395e23c1cd0f5b295bb97d88 Mon Sep 17 00:00:00 2001
From: Zile Xiong <xiongzile99 at gmail.com>
Date: Wed, 29 Apr 2026 00:25:23 +0800
Subject: [PATCH 1/3] [VPlan] Collect FOR PHIs before sinking/hoisting
recurrence users
Avoid iterating over HeaderVPBB->phis() while potentially mutating the
underlying VPBasicBlock. Collect all VPFirstOrderRecurrencePHIRecipe
instances first, then process them in a separate loop.
This prevents iterator invalidation when sinking or hoisting recurrence
users, and makes the transformation more robust.
Fixes: https://github.com/llvm/llvm-project/issues/194618
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 345bb2ef27a4f..391be24effd78 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -869,11 +869,13 @@ static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
/// fails.
static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
VPDominatorTree &VPDT) {
- for (VPRecipeBase &R : HeaderVPBB->phis()) {
- auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R);
- if (!FOR)
- continue;
+ SmallVector<VPFirstOrderRecurrencePHIRecipe *> FORs;
+
+ for (VPRecipeBase &R : HeaderVPBB->phis())
+ if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
+ FORs.push_back(FOR);
+ for (VPFirstOrderRecurrencePHIRecipe *FOR : FORs) {
// Follow through FOR phi chains to find the actual Previous recipe.
// Fixed-order recurrences do not contain cycles, so this loop is
// guaranteed to terminate.
>From c9bc8eddcfe8e146ccba8af6ef7df5002cf0d1d3 Mon Sep 17 00:00:00 2001
From: Zile Xiong <xiongzile99 at gmail.com>
Date: Wed, 29 Apr 2026 01:45:45 +0800
Subject: [PATCH 2/3] [VPlan] add test for crash
---
.../LoopVectorize/VPlan/for-phi-ordering.ll | 122 ++++++++++++++++++
1 file changed, 122 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
new file mode 100644
index 0000000000000..b2342e00a062e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -vplan-print-after=printOptimizedVPlan -disable-output 2>&1 | FileCheck %s
+ at b = global i32 0, align 4
+ at c = global i64 0, align 8
+ at g = global i32 0, align 4
+ at h = global i32 0, align 4
+ at f = global i32 0, align 4
+ at d = global i64 0, align 8
+
+define i32 @main() {
+; CHECK-LABEL: VPlan for loop in 'main'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %.pr = load i32, ptr @b, align 4
+; CHECK-NEXT: IR %h = load i32, ptr @h, align 4
+; CHECK-NEXT: IR %sext = shl i32 %h, 24
+; CHECK-NEXT: IR %tobool9 = icmp ne i32 %sext, 0
+; CHECK-NEXT: IR %conv8 = ashr exact i32 %sext, 24
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (0 smin %.pr)) + %.pr)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%.pr> + vp<[[VP2]]> * ir<-1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%.pr>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%rec> = phi ir<0>, ir<%dec>
+; CHECK-NEXT: WIDEN ir<%dec> = add nsw ir<%iv>, ir<-1>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: WIDEN ir<%tobool7> = icmp ne ir<%iv>, ir<0>
+; CHECK-NEXT: WIDEN ir<%or.cond> = select ir<%tobool7>, ir<%tobool9>, ir<false>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = select ir<%or.cond>, ir<1>, ir<%conv8>
+; CHECK-NEXT: WIDEN ir<%rem> = srem ir<%iv>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%cond> = ir<%rem> ir<0>/ir<%or.cond>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%rec>, ir<%dec>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = zext vp<[[VP8]]> to i64
+; CHECK-NEXT: EMIT vp<%vector.recur.extract.for.phi> = extract-penultimate-element ir<%dec>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part ir<%cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part ir<%conv>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %cond.lcssa = phi i32 [ %cond, %cond.end ] (extra operand: vp<[[VP10]]> from middle.block)
+; CHECK-NEXT: IR %dec.lcssa = phi i32 [ %dec, %cond.end ] (extra operand: vp<[[VP4]]> from middle.block)
+; CHECK-NEXT: IR %rec.lcssa = phi i32 [ %rec, %cond.end ] (extra operand: vp<%vector.recur.extract.for.phi> from middle.block)
+; CHECK-NEXT: IR %conv.lcssa = phi i64 [ %conv, %cond.end ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: IR %conv11 = sext i32 %cond.lcssa to i64
+; CHECK-NEXT: IR store i64 %conv.lcssa, ptr @c, align 8
+; CHECK-NEXT: IR store i32 %rec.lcssa, ptr @g, align 4
+; CHECK-NEXT: IR store i32 %cond.lcssa, ptr @f, align 4
+; CHECK-NEXT: IR store i64 %conv11, ptr @d, align 8
+; CHECK-NEXT: IR store i32 %dec.lcssa, ptr @b, align 4
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP4]]>, middle.block ], [ ir<%.pr>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ vp<[[VP4]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ %.pr, %entry ], [ %dec, %cond.end ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %rec = phi i32 [ 0, %entry ], [ %dec, %cond.end ] (extra operand: vp<%scalar.recur.init> from scalar.ph)
+; CHECK-NEXT: IR %conv = zext i32 %rec to i64
+; CHECK-NEXT: IR %tobool7 = icmp ne i32 %iv, 0
+; CHECK-NEXT: IR %or.cond = select i1 %tobool7, i1 %tobool9, i1 false
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %.pr = load i32, ptr @b, align 4
+ %h = load i32, ptr @h, align 4
+ %sext = shl i32 %h, 24
+ %tobool9 = icmp ne i32 %sext, 0
+ %conv8 = ashr exact i32 %sext, 24
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %.pr, %entry ], [ %dec, %cond.end ]
+ %rec = phi i32 [ 0, %entry ], [ %dec, %cond.end ]
+ %conv = zext i32 %rec to i64
+ %tobool7 = icmp ne i32 %iv, 0
+ %or.cond = select i1 %tobool7, i1 %tobool9, i1 false
+ br i1 %or.cond, label %cond.end, label %cond.false
+
+cond.false:
+ %rem = srem i32 %iv, %conv8
+ br label %cond.end
+
+cond.end:
+ %cond = phi i32 [ %rem, %cond.false ], [ 0, %loop ]
+ %dec = add nsw i32 %iv, -1
+ %cmp = icmp sgt i32 %iv, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ %conv11 = sext i32 %cond to i64
+ store i64 %conv, ptr @c, align 8
+ store i32 %rec, ptr @g, align 4
+ store i32 %cond, ptr @f, align 4
+ store i64 %conv11, ptr @d, align 8
+ store i32 %dec, ptr @b, align 4
+ ret i32 0
+}
>From b50e29aa748d3153c298a5c47b3afaededf7482f Mon Sep 17 00:00:00 2001
From: Zile Xiong <xiongzile99 at gmail.com>
Date: Wed, 29 Apr 2026 22:58:31 +0800
Subject: [PATCH 3/3] [VPlan] add test(v2)
---
.../LoopVectorize/VPlan/for-phi-ordering.ll | 75 ++++---------------
1 file changed, 14 insertions(+), 61 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
index b2342e00a062e..5a78298e093fd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/for-phi-ordering.ll
@@ -1,79 +1,42 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -vplan-print-after=printOptimizedVPlan -disable-output 2>&1 | FileCheck %s
- at b = global i32 0, align 4
- at c = global i64 0, align 8
- at g = global i32 0, align 4
- at h = global i32 0, align 4
- at f = global i32 0, align 4
- at d = global i64 0, align 8
-define i32 @main() {
-; CHECK-LABEL: VPlan for loop in 'main'
+define i32 @test(i32 %.pr, i1 %tobool9, i32 %conv8) {
+; CHECK-LABEL: VPlan for loop in 'test'
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
-; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: IR %.pr = load i32, ptr @b, align 4
-; CHECK-NEXT: IR %h = load i32, ptr @h, align 4
-; CHECK-NEXT: IR %sext = shl i32 %h, 24
-; CHECK-NEXT: IR %tobool9 = icmp ne i32 %sext, 0
-; CHECK-NEXT: IR %conv8 = ashr exact i32 %sext, 24
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (0 smin %.pr)) + %.pr)
+; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (0 smin %.pr)) + %.pr)
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%.pr> + vp<[[VP2]]> * ir<-1>
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<%.pr> + vp<[[VP1]]> * ir<-1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%.pr>, ir<-1>, vp<[[VP0]]>
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%rec> = phi ir<0>, ir<%dec>
-; CHECK-NEXT: WIDEN ir<%dec> = add nsw ir<%iv>, ir<-1>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: WIDEN ir<%tobool7> = icmp ne ir<%iv>, ir<0>
-; CHECK-NEXT: WIDEN ir<%or.cond> = select ir<%tobool7>, ir<%tobool9>, ir<false>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = select ir<%or.cond>, ir<1>, ir<%conv8>
-; CHECK-NEXT: WIDEN ir<%rem> = srem ir<%iv>, vp<[[VP7]]>
-; CHECK-NEXT: BLEND ir<%cond> = ir<%rem> ir<0>/ir<%or.cond>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%rec>, ir<%dec>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = zext vp<[[VP8]]> to i64
-; CHECK-NEXT: EMIT vp<%vector.recur.extract.for.phi> = extract-penultimate-element ir<%dec>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part ir<%cond>
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part ir<%conv>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP2]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %cond.lcssa = phi i32 [ %cond, %cond.end ] (extra operand: vp<[[VP10]]> from middle.block)
-; CHECK-NEXT: IR %dec.lcssa = phi i32 [ %dec, %cond.end ] (extra operand: vp<[[VP4]]> from middle.block)
-; CHECK-NEXT: IR %rec.lcssa = phi i32 [ %rec, %cond.end ] (extra operand: vp<%vector.recur.extract.for.phi> from middle.block)
-; CHECK-NEXT: IR %conv.lcssa = phi i64 [ %conv, %cond.end ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK-NEXT: IR %conv11 = sext i32 %cond.lcssa to i64
-; CHECK-NEXT: IR store i64 %conv.lcssa, ptr @c, align 8
-; CHECK-NEXT: IR store i32 %rec.lcssa, ptr @g, align 4
-; CHECK-NEXT: IR store i32 %cond.lcssa, ptr @f, align 4
-; CHECK-NEXT: IR store i64 %conv11, ptr @d, align 8
-; CHECK-NEXT: IR store i32 %dec.lcssa, ptr @b, align 4
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP4]]>, middle.block ], [ ir<%.pr>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ vp<[[VP4]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP3]]>, middle.block ], [ ir<%.pr>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ vp<[[VP3]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -86,11 +49,6 @@ define i32 @main() {
; CHECK-NEXT: }
;
entry:
- %.pr = load i32, ptr @b, align 4
- %h = load i32, ptr @h, align 4
- %sext = shl i32 %h, 24
- %tobool9 = icmp ne i32 %sext, 0
- %conv8 = ashr exact i32 %sext, 24
br label %loop
loop:
@@ -112,11 +70,6 @@ cond.end:
br i1 %cmp, label %loop, label %exit
exit:
- %conv11 = sext i32 %cond to i64
- store i64 %conv, ptr @c, align 8
- store i32 %rec, ptr @g, align 4
- store i32 %cond, ptr @f, align 4
- store i64 %conv11, ptr @d, align 8
- store i32 %dec, ptr @b, align 4
ret i32 0
}
+
More information about the llvm-commits
mailing list