[llvm] [VPlan] Collect FOR PHIs before sinking/hoisting recurrence users (PR #194671)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 08:30:45 PDT 2026
================
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -vplan-print-after=printOptimizedVPlan -disable-output 2>&1 | FileCheck %s
+ at b = global i32 0, align 4
+ at c = global i64 0, align 8
+ at g = global i32 0, align 4
+ at h = global i32 0, align 4
+ at f = global i32 0, align 4
+ at d = global i64 0, align 8
+
+define i32 @main() {
+; CHECK-LABEL: VPlan for loop in 'main'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %.pr = load i32, ptr @b, align 4
+; CHECK-NEXT: IR %h = load i32, ptr @h, align 4
+; CHECK-NEXT: IR %sext = shl i32 %h, 24
+; CHECK-NEXT: IR %tobool9 = icmp ne i32 %sext, 0
+; CHECK-NEXT: IR %conv8 = ashr exact i32 %sext, 24
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (0 smin %.pr)) + %.pr)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%.pr> + vp<[[VP2]]> * ir<-1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%.pr>, ir<-1>, vp<[[VP0]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%rec> = phi ir<0>, ir<%dec>
+; CHECK-NEXT: WIDEN ir<%dec> = add nsw ir<%iv>, ir<-1>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: WIDEN ir<%tobool7> = icmp ne ir<%iv>, ir<0>
+; CHECK-NEXT: WIDEN ir<%or.cond> = select ir<%tobool7>, ir<%tobool9>, ir<false>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = select ir<%or.cond>, ir<1>, ir<%conv8>
+; CHECK-NEXT: WIDEN ir<%rem> = srem ir<%iv>, vp<[[VP7]]>
+; CHECK-NEXT: BLEND ir<%cond> = ir<%rem> ir<0>/ir<%or.cond>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%rec>, ir<%dec>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = zext vp<[[VP8]]> to i64
+; CHECK-NEXT: EMIT vp<%vector.recur.extract.for.phi> = extract-penultimate-element ir<%dec>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part ir<%cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part ir<%conv>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %cond.lcssa = phi i32 [ %cond, %cond.end ] (extra operand: vp<[[VP10]]> from middle.block)
+; CHECK-NEXT: IR %dec.lcssa = phi i32 [ %dec, %cond.end ] (extra operand: vp<[[VP4]]> from middle.block)
+; CHECK-NEXT: IR %rec.lcssa = phi i32 [ %rec, %cond.end ] (extra operand: vp<%vector.recur.extract.for.phi> from middle.block)
+; CHECK-NEXT: IR %conv.lcssa = phi i64 [ %conv, %cond.end ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: IR %conv11 = sext i32 %cond.lcssa to i64
+; CHECK-NEXT: IR store i64 %conv.lcssa, ptr @c, align 8
+; CHECK-NEXT: IR store i32 %rec.lcssa, ptr @g, align 4
+; CHECK-NEXT: IR store i32 %cond.lcssa, ptr @f, align 4
+; CHECK-NEXT: IR store i64 %conv11, ptr @d, align 8
+; CHECK-NEXT: IR store i32 %dec.lcssa, ptr @b, align 4
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP4]]>, middle.block ], [ ir<%.pr>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%scalar.recur.init> = phi [ vp<[[VP4]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ %.pr, %entry ], [ %dec, %cond.end ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %rec = phi i32 [ 0, %entry ], [ %dec, %cond.end ] (extra operand: vp<%scalar.recur.init> from scalar.ph)
+; CHECK-NEXT: IR %conv = zext i32 %rec to i64
+; CHECK-NEXT: IR %tobool7 = icmp ne i32 %iv, 0
+; CHECK-NEXT: IR %or.cond = select i1 %tobool7, i1 %tobool9, i1 false
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ %.pr = load i32, ptr @b, align 4
+ %h = load i32, ptr @h, align 4
+ %sext = shl i32 %h, 24
+ %tobool9 = icmp ne i32 %sext, 0
+ %conv8 = ashr exact i32 %sext, 24
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %.pr, %entry ], [ %dec, %cond.end ]
----------------
xiongzile wrote:
Decrementing case can reproduce the crash because the ```moveAfter``` actually move the end of ```HeaderVPBB->phis()``` out of the block. See the log:
```C
move: EMIT-SCALAR ir<%conv> = zext ir<%rec> to i64
after: EMIT ir<%dec> = add nsw ir<%iv>, ir<-1>
moveAfter=================
loop:
EMIT-SCALAR vp<%4> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, cond.end ]
ir<%iv> = WIDEN-INDUCTION nsw ir<%.pr>, ir<-1>, vp<%0>
FIRST-ORDER-RECURRENCE-PHI ir<%rec> = phi ir<0>, ir<%dec>
EMIT-SCALAR ir<%conv> = zext ir<%rec> to i64 // Note: moved out !!!
EMIT ir<%tobool7> = icmp ne ir<%iv>, ir<0>
EMIT ir<%or.cond> = select ir<%tobool7>, ir<%tobool9>, ir<false>
EMIT branch-on-cond ir<%or.cond>
Successor(s): cond.end, cond.false
loop:
EMIT-SCALAR vp<%4> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, cond.end ]
ir<%iv> = WIDEN-INDUCTION nsw ir<%.pr>, ir<-1>, vp<%0>
FIRST-ORDER-RECURRENCE-PHI ir<%rec> = phi ir<0>, ir<%dec>
EMIT ir<%tobool7> = icmp ne ir<%iv>, ir<0>
EMIT ir<%or.cond> = select ir<%tobool7>, ir<%tobool9>, ir<false>
EMIT branch-on-cond ir<%or.cond>
Successor(s): cond.end, cond.false
moveAfter=================
```
The increasing-IV case I tried does not reproduce the crash, because the moved recipe remains in the same block and is still reachable from the current iterator.
See the log:
```C
move: EMIT-SCALAR ir<%use> = zext ir<%for.2> to i64
after: EMIT vp<%4> = first-order splice ir<%for.1>, ir<%for.1.next>
moveAfter=================
loop:
EMIT-SCALAR vp<%3> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, loop ]
EMIT ir<%p> = WIDEN-POINTER-INDUCTION ir<%ptr>, ir<4>, vp<%0>
FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<%4>
EMIT-SCALAR ir<%use> = zext ir<%for.2> to i64 // Note: I'm here!!!
EMIT-SCALAR ir<%for.1.next> = load ir<%p>
EMIT vp<%4> = first-order splice ir<%for.1>, ir<%for.1.next>
EMIT ir<%p.next> = getelementptr inbounds ir<%p>, ir<1>
EMIT ir<%exitcond.not> = icmp eq ir<%p.next>, ir<%end>
EMIT vp<%index.next> = add nuw vp<%3>, vp<%0>
EMIT branch-on-count vp<%index.next>, vp<%1>
Successor(s): middle.block, loop
loop:
EMIT-SCALAR vp<%3> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, loop ]
EMIT ir<%p> = WIDEN-POINTER-INDUCTION ir<%ptr>, ir<4>, vp<%0>
FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next>
FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<%4>
EMIT-SCALAR ir<%for.1.next> = load ir<%p>
EMIT vp<%4> = first-order splice ir<%for.1>, ir<%for.1.next>
EMIT-SCALAR ir<%use> = zext ir<%for.2> to i64 // Note: I'm moved here!!!
EMIT ir<%p.next> = getelementptr inbounds ir<%p>, ir<1>
EMIT ir<%exitcond.not> = icmp eq ir<%p.next>, ir<%end>
EMIT vp<%index.next> = add nuw vp<%3>, vp<%0>
EMIT branch-on-count vp<%index.next>, vp<%1>
Successor(s): middle.block, loop
moveAfter=================
```
https://github.com/llvm/llvm-project/pull/194671
More information about the llvm-commits
mailing list