[llvm] [LV] Support EVL for partial reduction and VPExpressionRecipe. (PR #205741)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 20:07:50 PDT 2026
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/205741
>From 433aaddeb86e5351d83bfa109f4fbf33dfa16c06 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 23 Jun 2026 23:05:44 -0700
Subject: [PATCH 01/13] precommit tests.
---
.../VPlan/RISCV/partial-reduce-dot-product.ll | 100 ++
.../VPlan/RISCV/reductions-evl.ll | 1258 +++++++++++++++++
2 files changed, 1358 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
new file mode 100644
index 0000000000000..582c5d3d783ee
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -scalable-vectorization=on -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s --check-prefixes=NON-VLA
+
+target triple = "riscv64-none-unknown-elf"
+
+define i32 @vdota4(ptr %a, ptr %b) #0 {
+; CHECK-LABEL: VPlan for loop in 'vdota4'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, vp<[[VP10:%[0-9]+]]>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<1024>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.a>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load.a> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = sext ir<%load.a> to i32
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
+; CHECK-NEXT: WIDEN ir<%add> = add ir<%mul>, ir<%accum>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP10]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%accum>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP11]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP10]]>
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<for.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv
+; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1
+; CHECK-NEXT: IR %ext.a = sext i8 %load.a to i32
+; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv
+; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1
+; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32
+; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a
+; CHECK-NEXT: IR %add = add i32 %mul, %accum
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ %gep.a = getelementptr i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %ext.a = sext i8 %load.a to i32
+ %gep.b = getelementptr i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul i32 %ext.b, %ext.a
+ %add = add i32 %mul, %accum
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit: ; preds = %for.body
+ ret i32 %add
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; NON-VLA: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
new file mode 100644
index 0000000000000..70b4226f02340
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -0,0 +1,1258 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -vplan-print-after="optimizeEVLMasks" -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -force-vector-width=4 -scalable-vectorization=on -prefer-inloop-reductions -disable-output %s 2>&1 | FileCheck %s
+
+; Tests for printing VPlans with reductions.
+
+define float @print_reduction(i64 %n, ptr noalias %y) {
+; CHECK-LABEL: VPlan for loop in 'print_reduction'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast vp.reduce.fadd (ir<%lv>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %red = phi float [ %red.next, %loop ], [ 0.000000e+00, %entry ] (extra operand: ir<0.000000e+00> from scalar.ph)
+; CHECK-NEXT: IR %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv
+; CHECK-NEXT: IR %lv = load float, ptr %arrayidx, align 4
+; CHECK-NEXT: IR %red.next = fadd fast float %lv, %red
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %exitcond = icmp eq i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop: ; preds = %entry, %loop
+ %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+ %red = phi float [ %red.next, %loop ], [ 0.0, %entry ]
+ %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv
+ %lv = load float, ptr %arrayidx, align 4
+ %red.next = fadd fast float %lv, %red
+ %iv.next = add i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit: ; preds = %loop, %entry
+ ret float %red.next
+}
+
+define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr noalias %dst) {
+; CHECK-LABEL: VPlan for loop in 'print_reduction_with_invariant_store'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast vp.reduce.fadd (ir<%lv>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
+; CHECK-NEXT: CLONE store vp<[[VP11]]>, ir<%dst>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %red = phi float [ %red.next, %loop ], [ 0.000000e+00, %entry ] (extra operand: ir<0.000000e+00> from scalar.ph)
+; CHECK-NEXT: IR %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv
+; CHECK-NEXT: IR %lv = load float, ptr %arrayidx, align 4
+; CHECK-NEXT: IR %red.next = fadd fast float %lv, %red
+; CHECK-NEXT: IR store float %red.next, ptr %dst, align 4
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %exitcond = icmp eq i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop: ; preds = %entry, %loop
+ %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+ %red = phi float [ %red.next, %loop ], [ 0.0, %entry ]
+ %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv
+ %lv = load float, ptr %arrayidx, align 4
+ %red.next = fadd fast float %lv, %red
+ store float %red.next, ptr %dst, align 4
+ %iv.next = add i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit: ; preds = %loop, %entry
+ ret void
+}
+
+define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'print_fmuladd_strict'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector reassoc nnan ninf nsz ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi (fmuladd) reassoc nnan ninf nsz vp<[[VP3]]>, ir<%muladd>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l.a> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l.b> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = fmul reassoc nnan ninf nsz ir<%l.a>, ir<%l.b>
+; CHECK-NEXT: REDUCE ir<%muladd> = ir<%sum.07> + reassoc nnan ninf nsz vp.reduce.fadd (vp<[[VP10]]>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (fmuladd, in-loop) reassoc nnan ninf nsz ir<%muladd>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %loop ] (extra operand: ir<0.000000e+00> from scalar.ph)
+; CHECK-NEXT: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+; CHECK-NEXT: IR %l.a = load float, ptr %arrayidx, align 4
+; CHECK-NEXT: IR %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv
+; CHECK-NEXT: IR %l.b = load float, ptr %arrayidx2, align 4
+; CHECK-NEXT: IR %muladd = tail call reassoc nnan ninf nsz float @llvm.fmuladd.f32(float %l.a, float %l.b, float %sum.07)
+; CHECK-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
+; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %loop ]
+ %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
+ %l.a = load float, ptr %arrayidx, align 4
+ %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv
+ %l.b = load float, ptr %arrayidx2, align 4
+ %muladd = tail call nnan ninf nsz reassoc float @llvm.fmuladd.f32(float %l.a, float %l.b, float %sum.07)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret float %muladd
+}
+
+define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
+; CHECK-LABEL: VPlan for loop in 'find_last_iv'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP5:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP5]]>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l.a> = vp.load vp<[[VP7]]>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%l.a>, ir<%start>
+; CHECK-NEXT: WIDEN ir<%cond> = select ir<%cmp2>, ir<%iv>, ir<%rdx>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<true>, ir<%cond>, ir<%rdx>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (smax) vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ne vp<[[VP11]]>, ir<-9223372036854775808>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = select vp<[[VP12]]>, vp<[[VP11]]>, ir<%start>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %inc, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %rdx = phi i64 [ %start, %entry ], [ %cond, %loop ] (extra operand: ir<%start> from scalar.ph)
+; CHECK-NEXT: IR %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
+; CHECK-NEXT: IR %l.a = load i64, ptr %gep.a, align 8
+; CHECK-NEXT: IR %cmp2 = icmp eq i64 %l.a, %start
+; CHECK-NEXT: IR %cond = select i1 %cmp2, i64 %iv, i64 %rdx
+; CHECK-NEXT: IR %inc = add nuw nsw i64 %iv, 1
+; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %inc, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+ %rdx = phi i64 [ %start, %entry ], [ %cond, %loop ]
+ %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
+ %l.a = load i64, ptr %gep.a, align 8
+ %cmp2 = icmp eq i64 %l.a, %start
+ %cond = select i1 %cmp2, i64 %iv, i64 %rdx
+ %inc = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %inc, %n
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret i64 %cond
+}
+
+define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'print_extended_reduction'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, ir<%rdx.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%conv0> = zext ir<%load0> to i64
+; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%conv0>, vp<%evl>)
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%rdx.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP9]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %arrayidx = getelementptr inbounds i32, ptr %x, i32 %iv
+; CHECK-NEXT: IR %load0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT: IR %conv0 = zext i32 %load0 to i64
+; CHECK-NEXT: IR %rdx.next = add nsw i64 %rdx, %conv0
+; CHECK-NEXT: IR %iv.next = add nuw nsw i32 %iv, 1
+; CHECK-NEXT: IR %exitcond = icmp eq i32 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+ %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds i32, ptr %x, i32 %iv
+ %load0 = load i32, ptr %arrayidx, align 4
+ %conv0 = zext i32 %load0 to i64
+ %rdx.next = add nsw i64 %rdx, %conv0
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %r.0.lcssa = phi i64 [ %rdx.next, %loop ]
+ ret i64 %r.0.lcssa
+}
+
+define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'print_mulacc'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, ir<%rdx.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load1> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%load0>, ir<%load1>
+; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%mul>, vp<%evl>)
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%rdx.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %arrayidx = getelementptr inbounds i64, ptr %x, i32 %iv
+; CHECK-NEXT: IR %load0 = load i64, ptr %arrayidx, align 8
+; CHECK-NEXT: IR %arrayidx1 = getelementptr inbounds i64, ptr %y, i32 %iv
+; CHECK-NEXT: IR %load1 = load i64, ptr %arrayidx1, align 8
+; CHECK-NEXT: IR %mul = mul nsw i64 %load0, %load1
+; CHECK-NEXT: IR %rdx.next = add nsw i64 %rdx, %mul
+; CHECK-NEXT: IR %iv.next = add nuw nsw i32 %iv, 1
+; CHECK-NEXT: IR %exitcond = icmp eq i32 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+ %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds i64, ptr %x, i32 %iv
+ %load0 = load i64, ptr %arrayidx, align 8
+ %arrayidx1 = getelementptr inbounds i64, ptr %y, i32 %iv
+ %load1 = load i64, ptr %arrayidx1, align 8
+ %mul = mul nsw i64 %load0, %load1
+ %rdx.next = add nsw i64 %rdx, %mul
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %r.0.lcssa = phi i64 [ %rdx.next, %loop ]
+ ret i64 %r.0.lcssa
+}
+
+define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, ir<%rdx.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load1> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%conv0> = sext ir<%load0> to i32
+; CHECK-NEXT: WIDEN-CAST ir<%conv1> = sext ir<%load1> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%conv0>, ir<%conv1>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
+; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%conv>, vp<%evl>)
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%rdx.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %arrayidx = getelementptr inbounds i16, ptr %x, i32 %iv
+; CHECK-NEXT: IR %load0 = load i16, ptr %arrayidx, align 8
+; CHECK-NEXT: IR %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %iv
+; CHECK-NEXT: IR %load1 = load i16, ptr %arrayidx1, align 8
+; CHECK-NEXT: IR %conv0 = sext i16 %load0 to i32
+; CHECK-NEXT: IR %conv1 = sext i16 %load1 to i32
+; CHECK-NEXT: IR %mul = mul nsw i32 %conv0, %conv1
+; CHECK-NEXT: IR %conv = sext i32 %mul to i64
+; CHECK-NEXT: IR %rdx.next = add nsw i64 %rdx, %conv
+; CHECK-NEXT: IR %iv.next = add nuw nsw i32 %iv, 1
+; CHECK-NEXT: IR %exitcond = icmp eq i32 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+ %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds i16, ptr %x, i32 %iv
+ %load0 = load i16, ptr %arrayidx, align 8
+ %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %iv
+ %load1 = load i16, ptr %arrayidx1, align 8
+ %conv0 = sext i16 %load0 to i32
+ %conv1 = sext i16 %load1 to i32
+ %mul = mul nsw i32 %conv0, %conv1
+ %conv = sext i32 %mul to i64
+ %rdx.next = add nsw i64 %rdx, %conv
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ %r.0.lcssa = phi i64 [ %rdx.next, %loop ]
+ ret i64 %r.0.lcssa
+}
+
+define i32 @print_mulacc_negated(ptr %a, ptr %b) {
+; CHECK-LABEL: VPlan for loop in 'print_mulacc_negated'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, ir<%add>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<1024>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.a>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load.a> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
+; CHECK-NEXT: WIDEN ir<%sub> = sub ir<0>, ir<%mul>
+; CHECK-NEXT: REDUCE ir<%add> = ir<%accum> + vp.reduce.add (ir<%sub>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP10]]>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%add>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv
+; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1
+; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i32
+; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv
+; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1
+; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32
+; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a
+; CHECK-NEXT: IR %sub = sub i32 0, %mul
+; CHECK-NEXT: IR %add = add i32 %accum, %sub
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %accum = phi i32 [ 0, %entry ], [ %add, %loop ]
+ %gep.a = getelementptr i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %ext.a = zext i8 %load.a to i32
+ %gep.b = getelementptr i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul i32 %ext.b, %ext.a
+ %sub = sub i32 0, %mul
+ %add = add i32 %accum, %sub
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ ret i32 %add
+}
+
+define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
+; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended_const'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63>
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
+; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
+; CHECK-NEXT: IR %l.ext = zext i8 %l to i32
+; CHECK-NEXT: IR %mul = mul i32 %l.ext, 63
+; CHECK-NEXT: IR %red.next = add i32 %red, %mul
+; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
+ %l = load i8, ptr %ptr.iv, align 1
+ %l.ext = zext i8 %l to i32
+ %mul = mul i32 %l.ext, 63
+ %red.next = add i32 %red, %mul
+ %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+ %ec = icmp eq ptr %ptr.iv, %end
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %red.next
+}
+
+define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
+; CHECK-LABEL: VPlan for loop in 'print_mulacc_extended_const_lhs'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<63>, ir<%l.ext>
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
+; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
+; CHECK-NEXT: IR %l.ext = zext i8 %l to i32
+; CHECK-NEXT: IR %mul = mul i32 63, %l.ext
+; CHECK-NEXT: IR %red.next = add i32 %red, %mul
+; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
+ %l = load i8, ptr %ptr.iv, align 1
+ %l.ext = zext i8 %l to i32
+ %mul = mul i32 63, %l.ext
+ %red.next = add i32 %red, %mul
+ %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+ %ec = icmp eq ptr %ptr.iv, %end
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %red.next
+}
+
+; Constants >= 128 cannot be treated as sign-extended, so the expression shouldn't extend 128
+define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
+; CHECK-LABEL: VPlan for loop in 'print_mulacc_not_extended_const'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (vp<[[VP10]]>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
+; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
+; CHECK-NEXT: IR %l.ext = sext i8 %l to i32
+; CHECK-NEXT: IR %mul = mul i32 %l.ext, 128
+; CHECK-NEXT: IR %red.next = add i32 %red, %mul
+; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
+ %l = load i8, ptr %ptr.iv, align 1
+ %l.ext = sext i8 %l to i32
+ %mul = mul i32 %l.ext, 128
+ %red.next = add i32 %red, %mul
+ %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+ %ec = icmp eq ptr %ptr.iv, %end
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %red.next.lcssa = phi i32 [ %red.next, %loop ]
+ ret i32 %red.next.lcssa
+}
+
+define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
+; CHECK-LABEL: VPlan for loop in 'print_ext_mulacc_extended_const'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63>
+; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul.ext>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
+; CHECK-NEXT: IR %red = phi i64 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
+; CHECK-NEXT: IR %l.ext = zext i8 %l to i32
+; CHECK-NEXT: IR %mul = mul i32 %l.ext, 63
+; CHECK-NEXT: IR %mul.ext = zext i32 %mul to i64
+; CHECK-NEXT: IR %red.next = add i64 %red, %mul.ext
+; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]
+ %red = phi i64 [ 0, %entry ], [ %red.next, %loop ]
+ %l = load i8, ptr %ptr.iv, align 1
+ %l.ext = zext i8 %l to i32
+ %mul = mul i32 %l.ext, 63
+ %mul.ext = zext i32 %mul to i64
+ %red.next = add i64 %red, %mul.ext
+ %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+ %ec = icmp eq ptr %ptr.iv, %end
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i64 %red.next
+}
+
+; Constants >= 128 cannot be treated as sign-extended, so the expression shouldn't extend 128
+define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
+; CHECK-LABEL: VPlan for loop in 'print_ext_mulacc_not_extended_const'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
+; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = sext vp<[[VP10]]> to i64
+; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul.ext>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
+; CHECK-NEXT: IR %red = phi i64 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
+; CHECK-NEXT: IR %l.ext = sext i8 %l to i32
+; CHECK-NEXT: IR %mul = mul i32 %l.ext, 128
+; CHECK-NEXT: IR %mul.ext = sext i32 %mul to i64
+; CHECK-NEXT: IR %red.next = add i64 %red, %mul.ext
+; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ]
+ %red = phi i64 [ 0, %entry ], [ %red.next, %loop ]
+ %l = load i8, ptr %ptr.iv, align 1
+ %l.ext = sext i8 %l to i32
+ %mul = mul i32 %l.ext, 128
+ %mul.ext = sext i32 %mul to i64
+ %red.next = add i64 %red, %mul.ext
+ %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
+ %ec = icmp eq ptr %ptr.iv, %end
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %red.next.lcssa = phi i64 [ %red.next, %loop ]
+ ret i64 %red.next.lcssa
+}
+
+; This reduce.add(ext(mul(ext(A), ext(B)))) can't be turned into an
+; ExtMulAccReduction VPExpressionRecipe since the mul has two users.
+; It can however be turned into an ExtendedReduction since that one doesn't
+; modify the mul's operands.
+define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
+; CHECK-LABEL: VPlan for loop in 'print_ext_mul_two_uses'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + %n)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%b> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%conv>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, ir<%add>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: REDUCE ir<%add> = ir<%res2> + vp.reduce.add (ir<%mul.ext>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP7]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%add>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %add.lcssa = phi i64 [ %add, %loop ] (extra operand: vp<[[VP9]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %res1 = phi i64 [ 0, %entry ], [ %load.ext.ext, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %res2 = phi i64 [ 0, %entry ], [ %add, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %load = load i16, ptr %a, align 2
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %conv = sext i16 %b to i32
+; CHECK-NEXT: IR %mul = mul i32 %conv, %conv
+; CHECK-NEXT: IR %mul.ext = zext i32 %mul to i64
+; CHECK-NEXT: IR %add = add i64 %res2, %mul.ext
+; CHECK-NEXT: IR %second_use = or i32 %mul, %c
+; CHECK-NEXT: IR %load.ext = sext i16 %load to i32
+; CHECK-NEXT: IR %load.ext.ext = sext i32 %load.ext to i64
+; CHECK-NEXT: IR %exitcond740.not = icmp eq i64 %iv, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %res1 = phi i64 [ 0, %entry ], [ %load.ext.ext, %loop ]
+ %res2 = phi i64 [ 0, %entry ], [ %add, %loop ]
+ %load = load i16, ptr %a, align 2
+ %iv.next = add i64 %iv, 1
+ %conv = sext i16 %b to i32
+ %mul = mul i32 %conv, %conv
+ %mul.ext = zext i32 %mul to i64
+ %add = add i64 %res2, %mul.ext
+ %second_use = or i32 %mul, %c ; this value is otherwise unused, but that's sufficient for the test
+ %load.ext = sext i16 %load to i32
+ %load.ext.ext = sext i32 %load.ext to i64
+ %exitcond740.not = icmp eq i64 %iv, %n
+ br i1 %exitcond740.not, label %exit, label %loop
+
+exit:
+ ret i64 %add
+}
+
+define i32 @print_umax_reduction(ptr %y) {
+; CHECK-LABEL: VPlan for loop in 'print_umax_reduction'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<100> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (umax) ir<0>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<100>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP5]]>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP7]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-INTRINSIC ir<%red.next> = call llvm.umax(ir<%lv>, ir<%red>)
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<true>, ir<%red.next>, ir<%red>, vp<%evl>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP9]]>, vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (umax) vp<[[VP8]]>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
+; CHECK-NEXT: IR %gep = getelementptr inbounds i32, ptr %y, i64 %iv
+; CHECK-NEXT: IR %lv = load i32, ptr %gep, align 4
+; CHECK-NEXT: IR %red.next = call i32 @llvm.umax.i32(i32 %lv, i32 %red)
+; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
+; CHECK-NEXT: IR %ec = icmp eq i64 %iv.next, 100
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %red.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %y, i64 %iv
+ %lv = load i32, ptr %gep, align 4
+ %red.next = call i32 @llvm.umax(i32 %lv, i32 %red)
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 100
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %red.next
+}
>From 3c84fff034cb5e99114bb20b045594a87954d961 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Wed, 24 Jun 2026 18:13:23 -0700
Subject: [PATCH 02/13] [LV] Support EVL partial reduction and
VPExpressionRecipe.
This patch adds the support for partial reduction with EVL tail-folding
by not creating a new VPExpressionEVLRecipes but just changing the last
expressionRecipes from VPReductionRecipe to VPReductionEVLRecipe.
Currently, all partial reductions will be converted to VPExpressionRecipe
when construction. So this patch also supports the VPExpressionRecipe
for EVL.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 12 +--
llvm/lib/Transforms/Vectorize/VPlan.h | 9 +-
.../Vectorize/VPlanEVLTailFolding.cpp | 6 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 91 +++++++++++++++----
.../RISCV/partial-reduce-dot-product.ll | 40 ++++----
.../VPlan/RISCV/partial-reduce-dot-product.ll | 10 +-
.../VPlan/RISCV/reductions-evl.ll | 63 ++++++-------
7 files changed, 142 insertions(+), 89 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c12a4f562f600..e24995d82c5ca 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6852,14 +6852,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
// Create partial reduction recipes for scaled reductions and transform
// recipes to abstract recipes if it is legal and beneficial and clamp the
// range for better cost estimation.
- // TODO: Enable following transform when the EVL-version of extended-reduction
- // and mulacc-reduction are implemented.
- if (!CM.foldTailWithEVL()) {
- RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
- Range);
- RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
- Range);
- }
+ RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
+ Range);
+ RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
+ Range);
// Interleave memory: for each Interleave Group we marked earlier as relevant
// for this VPlan, replace the Recipes widening its memory instructions with a
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 40c33de7504e7..bfde397c98dd0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3341,6 +3341,9 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
return Partial ? Partial->VFScaleFactor : 1;
}
+ /// The ReductionStyle of this recipe.
+ ReductionStyle getReductionStyle() const { return Style; }
+
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -3361,8 +3364,7 @@ class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
R.getFastMathFlagsOrNone(),
cast_or_null<Instruction>(R.getUnderlyingValue()),
{R.getChainOp(), R.getVecOp(), &EVL}, CondOp,
- getReductionStyle(/*InLoop=*/true, R.isOrdered(), 1),
- DL) {}
+ R.getReductionStyle(), DL) {}
~VPReductionEVLRecipe() override = default;
@@ -3667,6 +3669,9 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
}
+ /// Returns a new VPExpressionRecipe with VPReductionEVLRecipe.
+ VPExpressionRecipe *convertToEVL(VPValue &EVL, VPValue *Mask);
+
/// Insert the recipes of the expression back into the VPlan, directly before
/// the current recipe. Leaves the expression recipe empty, which must be
/// removed before codegen.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 42e8d87125f15..9cc4d14272960 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -237,6 +237,12 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
Mask ? Mask : Plan->getTrue(), &EVL},
IntrR->getScalarType(), {}, {}, DL);
+ // Transforms the VPReductionRecipe inside the VPExpressionRecipe.
+ if (auto *ExpressionR = dyn_cast<VPExpressionRecipe>(&CurRecipe))
+ if (match(ExpressionR->getOperand(ExpressionR->getNumOperands() - 1),
+ m_RemoveMask(HeaderMask, Mask)))
+ return ExpressionR->convertToEVL(EVL, Mask);
+
return nullptr;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b5524948c1bd9..862930d115973 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3429,7 +3429,7 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
Builder.setFastMathFlags(getFastMathFlagsOrNone());
RecurKind Kind = getRecurrenceKind();
- Value *Prev = State.get(getChainOp(), /*IsScalar*/ true);
+ Value *Prev = State.get(getChainOp(), !isPartialReduction());
Value *VecOp = State.get(getVecOp());
Value *EVL = State.get(getEVL(), VPLane(0));
@@ -3440,7 +3440,27 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
Value *NewRed;
- if (isOrdered()) {
+ if (isPartialReduction()) {
+ // For partial reduction, we need to generate the predicated select
+ // (vp.merge) since `@llvm.vector.partial.reduce()` doesn't have vector
+ // predicated version.
+ VectorType *VecTy = cast<VectorType>(VecOp->getType());
+ Value *Identity = getRecurrenceIdentity(Kind, VecTy->getElementType(),
+ getFastMathFlagsOrNone());
+ if (State.VF.isVector())
+ Identity =
+ State.Builder.CreateVectorSplat(VecTy->getElementCount(), Identity);
+
+ Value *NewVecOp = State.Builder.CreateIntrinsic(
+ VecTy, Intrinsic::vp_merge, {Mask, VecOp, Identity, EVL});
+ assert((Kind == RecurKind::Add || Kind == RecurKind::FAdd) &&
+ "Unexpected partial reduction kind");
+ NewRed = State.Builder.CreateIntrinsic(
+ Prev->getType(),
+ Kind == RecurKind::Add ? Intrinsic::vector_partial_reduce_add
+ : Intrinsic::vector_partial_reduce_fadd,
+ {Prev, NewVecOp}, State.Builder.getFastMathFlags(), "partial.reduce");
+ } else if (isOrdered()) {
NewRed = createOrderedReduction(Builder, Kind, VecOp, Prev, Mask, EVL);
} else {
NewRed = createSimpleReduction(Builder, VecOp, Kind, Mask, EVL);
@@ -3451,7 +3471,7 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
(Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(Kind), NewRed,
Prev);
}
- State.set(this, NewRed, /*IsScalar*/ true);
+ State.set(this, NewRed, !isPartialReduction());
}
InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
@@ -3559,6 +3579,32 @@ VPExpressionRecipe::VPExpressionRecipe(
R->replaceUsesOfWith(LiveIn, Tmp);
}
+VPExpressionRecipe *VPExpressionRecipe::convertToEVL(VPValue &EVL,
+ VPValue *Mask) {
+ // Clone the VPExpressionRecipe.
+ SmallVector<VPSingleDefRecipe *, 4> NewExpressionRecipes;
+ for (auto *R : ExpressionRecipes)
+ NewExpressionRecipes.push_back(R->clone());
+ for (auto *New : NewExpressionRecipes) {
+ for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
+ New->replaceUsesOfWith(Old, NewExpressionRecipes[Idx]);
+ // Update placeholder operands in the cloned recipe to use the external
+ // operands, to be internalized when the evl expression is constructed.
+ for (const auto &[Placeholder, OutsideOp] :
+ zip(LiveInPlaceholders, operands()))
+ New->replaceUsesOfWith(Placeholder, OutsideOp);
+ }
+
+ // Transforms to EVL reduction.
+ auto *Red = cast<VPReductionRecipe>(NewExpressionRecipes.pop_back_val());
+ if (!Mask)
+ Mask = getParent()->getPlan()->getTrue();
+ auto *NewRed = new VPReductionEVLRecipe(*Red, EVL, Mask, Red->getDebugLoc());
+ delete Red;
+ NewExpressionRecipes.push_back(NewRed);
+ return new VPExpressionRecipe(ExpressionType, NewExpressionRecipes);
+}
+
void VPExpressionRecipe::decompose() {
for (auto *R : ExpressionRecipes)
// Since the list could contain duplicates, make sure the recipe hasn't
@@ -3678,8 +3724,23 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << " = ";
auto *Red = cast<VPReductionRecipe>(ExpressionRecipes.back());
unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
- VPValue *RdxStart =
- getOperand(getNumOperands() - (Red->isConditional() ? 2 : 1));
+ VPValue *Mask = getOperand(getNumOperands() - 1);
+ VPValue *EVL =
+ isa<VPReductionEVLRecipe>(Red)
+ ? getOperand(getNumOperands() - (Red->isConditional() ? 2 : 1))
+ : nullptr;
+ VPValue *RdxStart = getOperand(
+ getNumOperands() - (Red->isConditional() ? 2 : 1) - (EVL ? 1 : 0));
+ auto PrintEVLAndMask = [&]() {
+ if (EVL) {
+ O << ", ";
+ EVL->printAsOperand(O, SlotTracker);
+ }
+ if (Red->isConditional()) {
+ O << ", ";
+ Mask->printAsOperand(O, SlotTracker);
+ }
+ };
switch (ExpressionType) {
case ExpressionTypes::NegatedExtendedReduction:
@@ -3698,10 +3759,7 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
auto *Ext0 = cast<VPWidenCastRecipe>(ExpressionRecipes[0]);
O << Instruction::getOpcodeName(Ext0->getOpcode()) << " to "
<< *Ext0->getScalarType();
- if (Red->isConditional()) {
- O << ", ";
- getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker);
- }
+ PrintEVLAndMask();
O << ")";
break;
}
@@ -3722,10 +3780,7 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
auto *Ext1 = cast<VPWidenCastRecipe>(ExpressionRecipes[1]);
O << " " << Instruction::getOpcodeName(Ext1->getOpcode()) << " to "
<< *Ext1->getScalarType() << ")";
- if (Red->isConditional()) {
- O << ", ";
- getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker);
- }
+ PrintEVLAndMask();
O << "))";
break;
}
@@ -3757,10 +3812,7 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << " " << Instruction::getOpcodeName(Ext1->getOpcode()) << " to "
<< *Ext1->getScalarType() << ")";
}
- if (Red->isConditional()) {
- O << ", ";
- getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker);
- }
+ PrintEVLAndMask();
O << ")";
break;
}
@@ -3791,7 +3843,10 @@ void VPReductionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
void VPReductionEVLRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
- O << Indent << "REDUCE ";
+ if (isPartialReduction())
+ O << Indent << "PARTIAL-REDUCE ";
+ else
+ O << Indent << "REDUCE ";
printAsOperand(O, SlotTracker);
O << " = ";
getChainOp()->printAsOperand(O, SlotTracker);
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index 85075addf1efa..d58d33e0208da 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -156,25 +156,25 @@ define i32 @vdot4a(ptr %a, ptr %b) #0 {
; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]]
; TAILFOLD: vector.body:
; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; TAILFOLD-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]]
-; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]]
-; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
+; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]])
; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; TAILFOLD: middle.block:
-; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]])
+; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]]
; TAILFOLD: for.exit:
; TAILFOLD-NEXT: ret i32 [[TMP10]]
@@ -349,25 +349,25 @@ define i32 @vdot4au(ptr %a, ptr %b) #0 {
; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]]
; TAILFOLD: vector.body:
; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; TAILFOLD-NEXT: [[TMP4:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]]
-; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]]
-; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
+; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]])
; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; TAILFOLD: middle.block:
-; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]])
+; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]]
; TAILFOLD: for.exit:
; TAILFOLD-NEXT: ret i32 [[TMP10]]
@@ -542,25 +542,25 @@ define i32 @vdot4asu(ptr %a, ptr %b) #0 {
; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]]
; TAILFOLD: vector.body:
; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; TAILFOLD-NEXT: [[TMP4:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; TAILFOLD-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]]
-; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]]
-; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
+; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]])
; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; TAILFOLD: middle.block:
-; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]])
+; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]]
; TAILFOLD: for.exit:
; TAILFOLD-NEXT: ret i32 [[TMP10]]
@@ -734,25 +734,25 @@ define i32 @vdot4asu2(ptr %a, ptr %b) #0 {
; TAILFOLD-NEXT: br label [[VECTOR_BODY:%.*]]
; TAILFOLD: vector.body:
; TAILFOLD-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
; TAILFOLD-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
; TAILFOLD-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
-; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
; TAILFOLD-NEXT: [[TMP4:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; TAILFOLD-NEXT: [[TMP2:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
; TAILFOLD-NEXT: [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], [[TMP2]]
-; TAILFOLD-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[TMP5]], [[VEC_PHI]]
-; TAILFOLD-NEXT: [[TMP7]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; TAILFOLD-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
+; TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP6]])
; TAILFOLD-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
; TAILFOLD-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP8]], [[EVL_BASED_IV]]
; TAILFOLD-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
; TAILFOLD-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
; TAILFOLD-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; TAILFOLD: middle.block:
-; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]])
+; TAILFOLD-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
; TAILFOLD-NEXT: br label [[FOR_EXIT:%.*]]
; TAILFOLD: for.exit:
; TAILFOLD-NEXT: ret i32 [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
index 582c5d3d783ee..5c761d465e01b 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -15,7 +15,7 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -23,7 +23,7 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, vp<[[VP10:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, vp<[[VP10:%[0-9]+]]> (VF scaled by 1/4)
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<1024>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
@@ -31,14 +31,10 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.a>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.a> = vp.load vp<[[VP8]]>, vp<%evl>
-; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = sext ir<%load.a> to i32
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl>
-; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
-; CHECK-NEXT: WIDEN ir<%add> = add ir<%mul>, ir<%accum>
-; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP10]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%accum>, vp<%evl>)
+; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> sext to i32), vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP11]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index 70b4226f02340..f2bbd10fbe16d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -342,15 +342,14 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, ir<%rdx.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
-; CHECK-NEXT: WIDEN-CAST ir<%conv0> = zext ir<%load0> to i64
-; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%conv0>, vp<%evl>)
+; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = ir<true> + reduce.add (ir<%load0> zext to i64, vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
@@ -360,11 +359,11 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%rdx.next>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP8]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP9]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -505,7 +504,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, ir<%rdx.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP9:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
@@ -518,8 +517,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: WIDEN-CAST ir<%conv0> = sext ir<%load0> to i32
; CHECK-NEXT: WIDEN-CAST ir<%conv1> = sext ir<%load1> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%conv0>, ir<%conv1>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
-; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%conv>, vp<%evl>)
+; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<true> + reduce.add (ir<%mul> sext to i64, vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
@@ -529,11 +527,11 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%rdx.next>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP9]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -940,7 +938,7 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, vp<[[VP10:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
@@ -950,11 +948,10 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63>
-; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
-; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul.ext>, vp<%evl>)
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
-; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<true> + reduce.add (ir<%mul> zext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
@@ -962,11 +959,11 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP10]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -1025,7 +1022,7 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
@@ -1035,11 +1032,10 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
-; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = sext vp<[[VP10]]> to i64
-; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + vp.reduce.add (ir<%mul.ext>, vp<%evl>)
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
-; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<true> + reduce.add (vp<[[VP10]]> sext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP12]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP12]]>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
@@ -1047,11 +1043,11 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -1109,7 +1105,6 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%b> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%conv>, ir<%conv>
-; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
@@ -1117,13 +1112,13 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, ir<%add>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, vp<[[VP7:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
-; CHECK-NEXT: REDUCE ir<%add> = ir<%res2> + vp.reduce.add (ir<%mul.ext>, vp<%evl>)
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
-; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP7]]>, vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP7]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP7]]> = ir<true> + reduce.add (ir<%mul> zext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP8]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP8]]>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
@@ -1131,11 +1126,11 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%add>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP7]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %add.lcssa = phi i64 [ %add, %loop ] (extra operand: vp<[[VP9]]> from middle.block)
+; CHECK-NEXT: IR %add.lcssa = phi i64 [ %add, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
>From 8871ee6b7d715533c851f70a7424ccf525fe0929 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Sun, 28 Jun 2026 22:25:18 -0700
Subject: [PATCH 03/13] Fixup, handle EVL conversion in VPlanTransforms.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 8 ++---
.../Vectorize/VPlanEVLTailFolding.cpp | 33 ++++++++++++++++---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 26 ---------------
.../VPlan/RISCV/partial-reduce-dot-product.ll | 6 ++--
.../VPlan/RISCV/reductions-evl.ll | 5 +++
5 files changed, 40 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bfde397c98dd0..1d291dc21bf8d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3587,6 +3587,7 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
/// Type of the expression.
ExpressionTypes ExpressionType;
+public:
/// Construct a new VPExpressionRecipe by internalizing recipes in \p
/// ExpressionRecipes. External operands (i.e. not defined by another recipe
/// in the expression) are replaced by temporary VPValues and the original
@@ -3596,7 +3597,6 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
VPExpressionRecipe(ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes);
-public:
VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
: VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {}
VPExpressionRecipe(VPWidenCastRecipe *Ext, VPWidenRecipe *Neg,
@@ -3669,14 +3669,14 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
}
- /// Returns a new VPExpressionRecipe with VPReductionEVLRecipe.
- VPExpressionRecipe *convertToEVL(VPValue &EVL, VPValue *Mask);
-
/// Insert the recipes of the expression back into the VPlan, directly before
/// the current recipe. Leaves the expression recipe empty, which must be
/// removed before codegen.
void decompose();
+ /// Returns the expression type of this recipe.
+ ExpressionTypes getExpressionType() const { return ExpressionType; }
+
unsigned getVFScaleFactor() const {
auto *PR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
return PR ? PR->getVFScaleFactor() : 1;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 9cc4d14272960..fe5aa88847c2a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -238,11 +238,34 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
IntrR->getScalarType(), {}, {}, DL);
// Transforms the VPReductionRecipe inside the VPExpressionRecipe.
- if (auto *ExpressionR = dyn_cast<VPExpressionRecipe>(&CurRecipe))
- if (match(ExpressionR->getOperand(ExpressionR->getNumOperands() - 1),
- m_RemoveMask(HeaderMask, Mask)))
- return ExpressionR->convertToEVL(EVL, Mask);
-
+ if (auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe))
+ if (match(Expr->getOperand(Expr->getNumOperands() - 1),
+ m_RemoveMask(HeaderMask, Mask))) {
+ auto PrevR = std::prev(Expr->getIterator());
+ Expr->decompose();
+ VPReductionRecipe *Red =
+ cast<VPReductionRecipe>(&*std::prev(Expr->getIterator()));
+
+ // Collect the recipes contained by the ExpressionRecipe except the last
+ // VPReductionRecipe.
+ SmallVector<VPSingleDefRecipe *, 4> Expressions;
+ for (auto &R :
+ make_range(std::next(PrevR), std::prev(Expr->getIterator())))
+ Expressions.push_back(cast<VPSingleDefRecipe>(&R));
+
+ // Convert to VPReductionEVLRecipe.
+ auto *NewRed = new VPReductionEVLRecipe(
+ *Red, EVL, Mask ? Mask : Plan->getTrue(), Red->getDebugLoc());
+ NewRed->insertBefore(Expr);
+ Expressions.push_back(NewRed);
+ auto *NewExpr =
+ new VPExpressionRecipe(Expr->getExpressionType(), Expressions);
+
+ // Replace uses and remove the old non-EVL reduction.
+ Red->replaceAllUsesWith(NewExpr);
+ Red->eraseFromParent();
+ return NewExpr;
+ }
return nullptr;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 862930d115973..5d8c1fbba23c5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3579,32 +3579,6 @@ VPExpressionRecipe::VPExpressionRecipe(
R->replaceUsesOfWith(LiveIn, Tmp);
}
-VPExpressionRecipe *VPExpressionRecipe::convertToEVL(VPValue &EVL,
- VPValue *Mask) {
- // Clone the VPExpressionRecipe.
- SmallVector<VPSingleDefRecipe *, 4> NewExpressionRecipes;
- for (auto *R : ExpressionRecipes)
- NewExpressionRecipes.push_back(R->clone());
- for (auto *New : NewExpressionRecipes) {
- for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
- New->replaceUsesOfWith(Old, NewExpressionRecipes[Idx]);
- // Update placeholder operands in the cloned recipe to use the external
- // operands, to be internalized when the evl expression is constructed.
- for (const auto &[Placeholder, OutsideOp] :
- zip(LiveInPlaceholders, operands()))
- New->replaceUsesOfWith(Placeholder, OutsideOp);
- }
-
- // Transforms to EVL reduction.
- auto *Red = cast<VPReductionRecipe>(NewExpressionRecipes.pop_back_val());
- if (!Mask)
- Mask = getParent()->getPlan()->getTrue();
- auto *NewRed = new VPReductionEVLRecipe(*Red, EVL, Mask, Red->getDebugLoc());
- delete Red;
- NewExpressionRecipes.push_back(NewRed);
- return new VPExpressionRecipe(ExpressionType, NewExpressionRecipes);
-}
-
void VPExpressionRecipe::decompose() {
for (auto *R : ExpressionRecipes)
// Since the list could contain duplicates, make sure the recipe hasn't
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
index 5c761d465e01b..5f1b2010d9a00 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -1,6 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=loop-vectorize -force-vector-width=4 -scalable-vectorization=on -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s
-; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s --check-prefixes=NON-VLA
target triple = "riscv64-none-unknown-elf"
@@ -34,6 +33,9 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
+; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = sext ir<%load.a> to i32
+; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> sext to i32), vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP11]]>, vp<[[VP5]]>
@@ -92,5 +94,3 @@ for.body: ; preds = %for.body, %entry
for.exit: ; preds = %for.body
ret i32 %add
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; NON-VLA: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index f2bbd10fbe16d..fc1496a82bcc5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -349,6 +349,7 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-CAST ir<%conv0> = zext ir<%load0> to i64
; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = ir<true> + reduce.add (ir<%load0> zext to i64, vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
@@ -517,6 +518,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: WIDEN-CAST ir<%conv0> = sext ir<%load0> to i32
; CHECK-NEXT: WIDEN-CAST ir<%conv1> = sext ir<%load1> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%conv0>, ir<%conv1>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<true> + reduce.add (ir<%mul> sext to i64, vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
@@ -948,6 +950,7 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63>
+; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<true> + reduce.add (ir<%mul> zext to i64, vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
@@ -1032,6 +1035,7 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
+; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = sext vp<[[VP10]]> to i64
; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<true> + reduce.add (vp<[[VP10]]> sext to i64, vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP12]]>, vp<[[VP6]]>
@@ -1115,6 +1119,7 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, vp<[[VP7:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
; CHECK-NEXT: EXPRESSION vp<[[VP7]]> = ir<true> + reduce.add (ir<%mul> zext to i64, vp<%evl>, ir<true>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP8]]>, vp<[[VP6]]>
>From 7c41f3bbe942322f80fdf020238afe28aa80ad03 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 29 Jun 2026 18:05:40 -0700
Subject: [PATCH 04/13] !fixup, returns expression recipes when decompose.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 8 ++++----
.../Vectorize/VPlanEVLTailFolding.cpp | 17 +++++------------
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +++-
3 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1d291dc21bf8d..5c144938dbc2f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3669,10 +3669,10 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
}
- /// Insert the recipes of the expression back into the VPlan, directly before
- /// the current recipe. Leaves the expression recipe empty, which must be
- /// removed before codegen.
- void decompose();
+ /// Return and insert the recipes of the expression back into the VPlan,
+ /// directly before the current recipe. Leaves the expression recipe empty,
+ /// which must be removed before codegen.
+ ArrayRef<VPSingleDefRecipe *> decompose();
/// Returns the expression type of this recipe.
ExpressionTypes getExpressionType() const { return ExpressionType; }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index fe5aa88847c2a..047ae7ae86032 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -241,25 +241,18 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
if (auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe))
if (match(Expr->getOperand(Expr->getNumOperands() - 1),
m_RemoveMask(HeaderMask, Mask))) {
- auto PrevR = std::prev(Expr->getIterator());
- Expr->decompose();
+ // Decompose first and construct with VPReductionEVLRecipe later.
+ SmallVector<VPSingleDefRecipe *> ExpressionRecipes(Expr->decompose());
VPReductionRecipe *Red =
- cast<VPReductionRecipe>(&*std::prev(Expr->getIterator()));
-
- // Collect the recipes contained by the ExpressionRecipe except the last
- // VPReductionRecipe.
- SmallVector<VPSingleDefRecipe *, 4> Expressions;
- for (auto &R :
- make_range(std::next(PrevR), std::prev(Expr->getIterator())))
- Expressions.push_back(cast<VPSingleDefRecipe>(&R));
+ cast<VPReductionRecipe>(ExpressionRecipes.pop_back_val());
// Convert to VPReductionEVLRecipe.
auto *NewRed = new VPReductionEVLRecipe(
*Red, EVL, Mask ? Mask : Plan->getTrue(), Red->getDebugLoc());
NewRed->insertBefore(Expr);
- Expressions.push_back(NewRed);
+ ExpressionRecipes.push_back(NewRed);
auto *NewExpr =
- new VPExpressionRecipe(Expr->getExpressionType(), Expressions);
+ new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes);
// Replace uses and remove the old non-EVL reduction.
Red->replaceAllUsesWith(NewExpr);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5d8c1fbba23c5..02c348fed9ad0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3579,7 +3579,7 @@ VPExpressionRecipe::VPExpressionRecipe(
R->replaceUsesOfWith(LiveIn, Tmp);
}
-void VPExpressionRecipe::decompose() {
+ArrayRef<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
for (auto *R : ExpressionRecipes)
// Since the list could contain duplicates, make sure the recipe hasn't
// already been inserted.
@@ -3590,7 +3590,9 @@ void VPExpressionRecipe::decompose() {
LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
replaceAllUsesWith(ExpressionRecipes.back());
+ ArrayRef<VPSingleDefRecipe *> Expressions(ExpressionRecipes);
ExpressionRecipes.clear();
+ return Expressions;
}
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
>From 2c613393d801646661a0c09a5114ec3a565d6bcd Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 30 Jun 2026 16:25:24 -0700
Subject: [PATCH 05/13] Address comments.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 19 +++++-----
.../Vectorize/VPlanEVLTailFolding.cpp | 17 +--------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 36 +++++++++++++++----
.../RISCV/partial-reduce-dot-product.ll | 2 --
.../VPlan/RISCV/partial-reduce-dot-product.ll | 9 ++---
.../VPlan/RISCV/reductions-evl.ll | 15 +++-----
6 files changed, 47 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5c144938dbc2f..34c1deaba597c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3341,9 +3341,6 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
return Partial ? Partial->VFScaleFactor : 1;
}
- /// The ReductionStyle of this recipe.
- ReductionStyle getReductionStyle() const { return Style; }
-
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -3364,7 +3361,9 @@ class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
R.getFastMathFlagsOrNone(),
cast_or_null<Instruction>(R.getUnderlyingValue()),
{R.getChainOp(), R.getVecOp(), &EVL}, CondOp,
- R.getReductionStyle(), DL) {}
+ getReductionStyle(R.isInLoop(), R.isOrdered(),
+ R.getVFScaleFactor()),
+ DL) {}
~VPReductionEVLRecipe() override = default;
@@ -3669,13 +3668,13 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
}
- /// Return and insert the recipes of the expression back into the VPlan,
- /// directly before the current recipe. Leaves the expression recipe empty,
- /// which must be removed before codegen.
- ArrayRef<VPSingleDefRecipe *> decompose();
+ /// Clone this recipe and replace the reduction recipe to EVL recipe.
+ VPExpressionRecipe *cloneWithEVL(VPValue *Mask, VPValue *EVL);
- /// Returns the expression type of this recipe.
- ExpressionTypes getExpressionType() const { return ExpressionType; }
+ /// Insert the recipes of the expression back into the VPlan,
+ /// directly before the current recipe. Leaves the expression
+ /// recipe empty, which must be removed before codegen.
+ void decompose();
unsigned getVFScaleFactor() const {
auto *PR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 047ae7ae86032..fff6457137fea 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -241,22 +241,7 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
if (auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe))
if (match(Expr->getOperand(Expr->getNumOperands() - 1),
m_RemoveMask(HeaderMask, Mask))) {
- // Decompose first and construct with VPReductionEVLRecipe later.
- SmallVector<VPSingleDefRecipe *> ExpressionRecipes(Expr->decompose());
- VPReductionRecipe *Red =
- cast<VPReductionRecipe>(ExpressionRecipes.pop_back_val());
-
- // Convert to VPReductionEVLRecipe.
- auto *NewRed = new VPReductionEVLRecipe(
- *Red, EVL, Mask ? Mask : Plan->getTrue(), Red->getDebugLoc());
- NewRed->insertBefore(Expr);
- ExpressionRecipes.push_back(NewRed);
- auto *NewExpr =
- new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes);
-
- // Replace uses and remove the old non-EVL reduction.
- Red->replaceAllUsesWith(NewExpr);
- Red->eraseFromParent();
+ auto *NewExpr = Expr->cloneWithEVL(Mask, &EVL);
return NewExpr;
}
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 02c348fed9ad0..5519d48893ead 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3423,6 +3423,8 @@ void VPReductionRecipe::execute(VPTransformState &State) {
void VPReductionEVLRecipe::execute(VPTransformState &State) {
+ assert(State.VF.isVector() &&
+ "Shouldn't generate VPReductionEVLRecipe with scalar VF");
auto &Builder = State.Builder;
// Propagate the fast-math flags carried by the underlying instruction.
IRBuilderBase::FastMathFlagGuard FMFGuard(Builder);
@@ -3447,9 +3449,8 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
VectorType *VecTy = cast<VectorType>(VecOp->getType());
Value *Identity = getRecurrenceIdentity(Kind, VecTy->getElementType(),
getFastMathFlagsOrNone());
- if (State.VF.isVector())
- Identity =
- State.Builder.CreateVectorSplat(VecTy->getElementCount(), Identity);
+ Identity =
+ State.Builder.CreateVectorSplat(VecTy->getElementCount(), Identity);
Value *NewVecOp = State.Builder.CreateIntrinsic(
VecTy, Intrinsic::vp_merge, {Mask, VecOp, Identity, EVL});
@@ -3579,7 +3580,7 @@ VPExpressionRecipe::VPExpressionRecipe(
R->replaceUsesOfWith(LiveIn, Tmp);
}
-ArrayRef<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
+void VPExpressionRecipe::decompose() {
for (auto *R : ExpressionRecipes)
// Since the list could contain duplicates, make sure the recipe hasn't
// already been inserted.
@@ -3590,9 +3591,32 @@ ArrayRef<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
replaceAllUsesWith(ExpressionRecipes.back());
- ArrayRef<VPSingleDefRecipe *> Expressions(ExpressionRecipes);
ExpressionRecipes.clear();
- return Expressions;
+}
+
+VPExpressionRecipe *VPExpressionRecipe::cloneWithEVL(VPValue *Mask,
+ VPValue *EVL) {
+ assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
+ SmallVector<VPSingleDefRecipe *> NewExpressiondRecipes;
+ for (auto *R : ExpressionRecipes)
+ NewExpressiondRecipes.push_back(R->clone());
+
+ for (auto *New : NewExpressiondRecipes) {
+ for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
+ New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
+ // Update placeholder operands in the cloned recipe to use the external
+ // operands, to be internalized when the cloned expression is constructed.
+ for (const auto &[Placeholder, OutsideOp] :
+ zip(LiveInPlaceholders, operands()))
+ New->replaceUsesOfWith(Placeholder, OutsideOp);
+ }
+
+ // Convert to VPReductionEVLRecipe
+ auto *Red = cast<VPReductionRecipe>(NewExpressiondRecipes.pop_back_val());
+ auto *NewRed = new VPReductionEVLRecipe(*Red, *EVL, Mask);
+ delete Red;
+ NewExpressiondRecipes.push_back(NewRed);
+ return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
}
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index d58d33e0208da..c07b7d1f91fa9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -5,8 +5,6 @@
; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -scalable-vectorization=off -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefixes=FIXED,FIXED-ZVDOT4A8I
; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -S < %s | FileCheck %s --check-prefixes=CHECK,TAILFOLD
-; TODO: Remove -tail-folding-policy=dont-fold-tail when partial reductions with EVL tail folding is supported.
-
target triple = "riscv64-none-unknown-elf"
define i32 @vdot4a(ptr %a, ptr %b) #0 {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
index 5f1b2010d9a00..210e38c53e965 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -1,7 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -scalable-vectorization=on -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s
-
-target triple = "riscv64-none-unknown-elf"
+; RUN: opt -mtriple riscv64 -passes=loop-vectorize -force-vector-width=4 -scalable-vectorization=on -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s
define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-LABEL: VPlan for loop in 'vdota4'
@@ -33,10 +31,7 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl>
-; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
-; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = sext ir<%load.a> to i32
-; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
-; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> sext to i32), vp<%evl>, ir<true>)
+; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> sext to i32), vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP11]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index fc1496a82bcc5..5b1b19fffd7ec 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -349,8 +349,7 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
-; CHECK-NEXT: WIDEN-CAST ir<%conv0> = zext ir<%load0> to i64
-; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = ir<true> + reduce.add (ir<%load0> zext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = vp<%evl> + reduce.add (ir<%load0> zext to i64, vp<%evl>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
@@ -518,8 +517,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: WIDEN-CAST ir<%conv0> = sext ir<%load0> to i32
; CHECK-NEXT: WIDEN-CAST ir<%conv1> = sext ir<%load1> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%conv0>, ir<%conv1>
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
-; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<true> + reduce.add (ir<%mul> sext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = vp<%evl> + reduce.add (ir<%mul> sext to i64, vp<%evl>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
@@ -950,8 +948,7 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63>
-; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
-; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<true> + reduce.add (ir<%mul> zext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = vp<%evl> + reduce.add (ir<%mul> zext to i64, vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
@@ -1035,8 +1032,7 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
-; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = sext vp<[[VP10]]> to i64
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<true> + reduce.add (vp<[[VP10]]> sext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = vp<%evl> + reduce.add (vp<[[VP10]]> sext to i64, vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP12]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP12]]>
@@ -1119,8 +1115,7 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, vp<[[VP7:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
-; CHECK-NEXT: WIDEN-CAST ir<%mul.ext> = zext ir<%mul> to i64
-; CHECK-NEXT: EXPRESSION vp<[[VP7]]> = ir<true> + reduce.add (ir<%mul> zext to i64, vp<%evl>, ir<true>)
+; CHECK-NEXT: EXPRESSION vp<[[VP7]]> = vp<%evl> + reduce.add (ir<%mul> zext to i64, vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP8]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP8]]>
>From 3e74d6fcc4d222075084c73f5b3dd4bfadf2e757 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 7 Jul 2026 19:42:55 -0700
Subject: [PATCH 06/13] Add predicate partial reduction tests.
---
.../RISCV/partial-reduce-dot-product.ll | 2 +
.../RISCV/partial-reduce-with-predicate.ll | 858 ++++++++++++++++++
2 files changed, 860 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index c07b7d1f91fa9..d58d33e0208da 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -5,6 +5,8 @@
; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -scalable-vectorization=off -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefixes=FIXED,FIXED-ZVDOT4A8I
; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvdot4a8i -S < %s | FileCheck %s --check-prefixes=CHECK,TAILFOLD
+; TODO: Remove -tail-folding-policy=dont-fold-tail when partial reductions with EVL tail folding is supported.
+
target triple = "riscv64-none-unknown-elf"
define i32 @vdot4a(ptr %a, ptr %b) #0 {
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
new file mode 100644
index 0000000000000..80c6ebe61c268
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
@@ -0,0 +1,858 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -S < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -scalable-vectorization=off -tail-folding-policy=dont-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-FIXED
+
+define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @pred_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PARTIAL_REDUCE4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]]
+; CHECK-NEXT: [[TMP6]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret i32 [[TMP9]]
+;
+; CHECK-FIXED-LABEL: define i32 @pred_reduction(
+; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-FIXED: [[VECTOR_PH]]:
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-FIXED: [[VECTOR_BODY]]:
+; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-FIXED-NEXT: [[TMP12:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]]
+; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]]
+; CHECK-FIXED-NEXT: [[PREDPHI4]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP12]], <8 x i32> [[VEC_PHI1]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-FIXED: [[MIDDLE_BLOCK]]:
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI4]], [[PREDPHI]]
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-FIXED: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @pred_reduction_sext(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[PARTIAL_REDUCE4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = sext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]]
+; CHECK-NEXT: [[TMP6]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[PARTIAL_REDUCE4]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret i32 [[TMP9]]
+;
+; CHECK-FIXED-LABEL: define i32 @pred_reduction_sext(
+; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-FIXED: [[VECTOR_PH]]:
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-FIXED: [[VECTOR_BODY]]:
+; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = sext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-FIXED-NEXT: [[TMP12:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]]
+; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]]
+; CHECK-FIXED-NEXT: [[PREDPHI4]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP12]], <8 x i32> [[VEC_PHI1]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-FIXED: [[MIDDLE_BLOCK]]:
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI4]], [[PREDPHI]]
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-FIXED: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = sext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP10]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret i32 [[TMP11]]
+;
+; CHECK-FIXED-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-FIXED: [[VECTOR_PH]]:
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-FIXED: [[VECTOR_BODY]]:
+; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP7]], <8 x i1> [[TMP2]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP16]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD4]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = mul nuw nsw <8 x i32> [[TMP8]], [[TMP9]]
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP10]], <8 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP11]])
+; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP14:%.*]] = mul nuw nsw <8 x i32> [[TMP18]], [[TMP13]]
+; CHECK-FIXED-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP14]], <8 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP15]])
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-FIXED: [[MIDDLE_BLOCK]]:
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-FIXED: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %arrayidx2, align 1
+ %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %arrayidx4, align 1
+ %ext.a = zext i8 %load.a to i32
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul nuw nsw i32 %ext.a, %ext.b
+ %add = add nsw i32 %sum, %mul
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @pred_sub_reduction(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 1 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP10]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = mul nuw nsw <vscale x 4 x i32> [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP11]])
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv1i32(<vscale x 1 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: [[TMP22:%.*]] = sub i32 0, [[TMP21]]
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret i32 [[TMP22]]
+;
+; CHECK-FIXED-LABEL: define i32 @pred_sub_reduction(
+; CHECK-FIXED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-FIXED: [[VECTOR_PH]]:
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-FIXED: [[VECTOR_BODY]]:
+; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP7]], <8 x i1> [[TMP2]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP16]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD4]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = mul nuw nsw <8 x i32> [[TMP8]], [[TMP9]]
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP10]], <8 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI]], <8 x i32> [[TMP11]])
+; CHECK-FIXED-NEXT: [[TMP17:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP19:%.*]] = mul nuw nsw <8 x i32> [[TMP17]], [[TMP18]]
+; CHECK-FIXED-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-FIXED-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI1]], <8 x i32> [[TMP15]])
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-FIXED: [[MIDDLE_BLOCK]]:
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <2 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-FIXED-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
+; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-FIXED: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %arrayidx2, align 1
+ %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %arrayidx4, align 1
+ %ext.a = zext i8 %load.a to i32
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul nuw nsw i32 %ext.a, %ext.b
+ %sub = sub nsw i32 %sum, %mul
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %sub, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @chained_pred_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD1]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP6]]
+; CHECK-NEXT: [[PARTIAL_REDUCE8:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE7:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP9]])
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret i32 [[TMP12]]
+;
+; CHECK-FIXED-LABEL: define i32 @chained_pred_reduction(
+; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-FIXED: [[VECTOR_PH]]:
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-FIXED: [[VECTOR_BODY]]:
+; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP6]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP6]], <8 x i1> [[TMP2]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP5]], <8 x i1> [[TMP3]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD3]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP10]]
+; CHECK-FIXED-NEXT: [[TMP16:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP7]]
+; CHECK-FIXED-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP8]], <8 x i32> [[VEC_PHI]]
+; CHECK-FIXED-NEXT: [[PREDPHI4:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP16]], <8 x i32> [[VEC_PHI1]]
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP9]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP11]], align 1
+; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_LOAD5]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP14]] = add <8 x i32> [[PREDPHI]], [[TMP18]]
+; CHECK-FIXED-NEXT: [[TMP15]] = add <8 x i32> [[PREDPHI4]], [[TMP13]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-FIXED: [[MIDDLE_BLOCK]]:
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[TMP15]], [[TMP14]]
+; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-FIXED: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.2 = add nsw i32 %sum.1, %bconv
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
+define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @reduction_before_pred(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i8> [[VP_OP_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr align 1 [[TMP12]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
+; CHECK-NEXT: [[PARTIAL_REDUCE7:%.*]] = zext <vscale x 4 x i8> [[VP_OP_LOAD2]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> [[BIN_RDX]], <vscale x 4 x i32> [[PARTIAL_REDUCE8]]
+; CHECK-NEXT: [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP9]])
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: ret i32 [[TMP13]]
+;
+; CHECK-FIXED-LABEL: define i32 @reduction_before_pred(
+; CHECK-FIXED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-FIXED-NEXT: [[ENTRY:.*:]]
+; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-FIXED: [[VECTOR_PH]]:
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-FIXED: [[VECTOR_BODY]]:
+; CHECK-FIXED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[VEC_PHI1:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-FIXED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
+; CHECK-FIXED-NEXT: [[TMP2:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[WIDE_LOAD2]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP6:%.*]] = add <8 x i32> [[VEC_PHI]], [[TMP2]]
+; CHECK-FIXED-NEXT: [[TMP5:%.*]] = add <8 x i32> [[VEC_PHI1]], [[TMP3]]
+; CHECK-FIXED-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i8>, ptr [[TMP7]], align 1
+; CHECK-FIXED-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP10]], align 1
+; CHECK-FIXED-NEXT: [[TMP8:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD3]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP16:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD4]], zeroinitializer
+; CHECK-FIXED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-FIXED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP9]], i64 8
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP9]], <8 x i1> [[TMP8]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP11]], <8 x i1> [[TMP16]], <8 x i8> poison)
+; CHECK-FIXED-NEXT: [[TMP18:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP13:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD5]] to <8 x i32>
+; CHECK-FIXED-NEXT: [[TMP14:%.*]] = add <8 x i32> [[TMP6]], [[TMP18]]
+; CHECK-FIXED-NEXT: [[TMP15:%.*]] = add <8 x i32> [[TMP5]], [[TMP13]]
+; CHECK-FIXED-NEXT: [[PREDPHI]] = select <8 x i1> [[TMP8]], <8 x i32> [[TMP14]], <8 x i32> [[TMP6]]
+; CHECK-FIXED-NEXT: [[PREDPHI6]] = select <8 x i1> [[TMP16]], <8 x i32> [[TMP15]], <8 x i32> [[TMP5]]
+; CHECK-FIXED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-FIXED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-FIXED: [[MIDDLE_BLOCK]]:
+; CHECK-FIXED-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[PREDPHI6]], [[PREDPHI]]
+; CHECK-FIXED-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[BIN_RDX]])
+; CHECK-FIXED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-FIXED-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-FIXED: [[SCALAR_PH]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.1 = add nsw i32 %sum, %bconv
+
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum.1, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.2 = phi i32 [ %sum.1, %for.body ], [ %add, %if.then ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
+; UTC_ARGS: --disable
+
+; Negative test - expect not to generate a partial reduction as phi
+; has two update values
+define i32 @partial_reduce_if_else(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @partial_reduce_if_else(
+; CHECK-NOT: llvm.vector.partial.reduce
+; CHECK: call i32 @llvm.vector.reduce.add
+;
+; CHECK-FIXED-LABEL: define i32 @partial_reduce_if_else(
+; CHECK-FIXED-NOT: llvm.vector.partial.reduce
+; CHECK-FIXED: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.inc ]
+
+ %arrayidx.cond = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx.cond, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %if.else, label %if.then
+
+if.then:
+ %arrayidx.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %aval = load i8, ptr %arrayidx.a, align 1
+ %a.ext = zext i8 %aval to i32
+ %add.a = add nsw i32 %sum, %a.ext
+ br label %for.inc
+
+if.else:
+ %arrayidx.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %bval = load i8, ptr %arrayidx.b, align 1
+ %b.ext = zext i8 %bval to i32
+ %add.b = add nsw i32 %sum, %b.ext
+ br label %for.inc
+
+for.inc:
+ %sum.next = phi i32 [ %add.a, %if.then ], [ %add.b, %if.else ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.next
+}
+
+; Negative test - expect not to generate the partial reduction as
+; there are more than two incomng values to the predicated phi.
+define i32 @partial_reduce_three_incoming(ptr %a, ptr %b, ptr %cond_a, ptr %cond_b, i64 %N) #0 {
+; CHECK-LABEL: define i32 @partial_reduce_three_incoming(
+; CHECK-NOT: llvm.vector.partial.reduce
+; CHECK: call i32 @llvm.vector.reduce.add
+;
+; CHECK-FIXED-LABEL: define i32 @partial_reduce_three_incoming(
+; CHECK-FIXED-NOT: llvm.vector.partial.reduce
+; CHECK-FIXED: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.inc ]
+
+ %arrayidx.cond.a = getelementptr inbounds nuw i8, ptr %cond_a, i64 %iv
+ %ca = load i8, ptr %arrayidx.cond.a, align 1
+ %cond.a = icmp ne i8 %ca, 0
+ br i1 %cond.a, label %if.then, label %if.else
+
+if.then:
+ %arrayidx.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %aval = load i8, ptr %arrayidx.a, align 1
+ %a.ext = zext i8 %aval to i32
+ %add.a = add nsw i32 %sum, %a.ext
+ br label %for.inc
+
+if.else:
+ %arrayidx.cond.b = getelementptr inbounds nuw i8, ptr %cond_b, i64 %iv
+ %cb = load i8, ptr %arrayidx.cond.b, align 1
+ %cond.b = icmp ne i8 %cb, 0
+ br i1 %cond.b, label %if.else.then, label %if.else.end
+
+if.else.then:
+ %arrayidx.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %bval = load i8, ptr %arrayidx.b, align 1
+ %b.ext = zext i8 %bval to i32
+ %add.b = add nsw i32 %sum, %b.ext
+ br label %for.inc
+
+if.else.end:
+ br label %for.inc
+
+for.inc:
+ %sum.next = phi i32 [ %add.a, %if.then ],
+ [ %add.b, %if.else.then ],
+ [ %sum, %if.else.end ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.next
+}
+
+; Negative test - chained reduction with mixed scale factors
+define i32 @partial_reduce_mixed_scale_factor(ptr %a8, ptr %b16,
+ ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_mixed_scale_factor(
+; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
+;
+; CHECK-FIXED-LABEL: define i32 @partial_reduce_mixed_scale_factor(
+; CHECK-FIXED-NOT: llvm.vector.partial.reduce
+; CHECK-FIXED: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+
+ %a.gep = getelementptr inbounds nuw i8, ptr %a8, i64 %iv
+ %aval = load i8, ptr %a.gep, align 1
+ %a.ext = zext i8 %aval to i32
+ %sum.1 = add nsw i32 %sum, %a.ext
+
+ %cond.gep = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %cond.gep, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %b.gep = getelementptr inbounds nuw i16, ptr %b16, i64 %iv
+ %bval = load i16, ptr %b.gep, align 2
+ %b.ext = zext i16 %bval to i32
+ %add = add nsw i32 %sum.1, %b.ext
+ br label %for.inc
+
+for.inc:
+ %sum.2 = phi i32 [ %add, %if.then ], [ %sum.1, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
+; Negative test - the incoming value doesn't match the accumulator
+; the passthrough value is %sum, not the previous chain value %sum.1.
+define i32 @partial_reduce_bad_passthrough(ptr %src, ptr noalias %src_b,
+ ptr %cond, i64 %N) #0 {
+; CHECK-LABEL: define i32 @partial_reduce_bad_passthrough(
+; CHECK-NOT: llvm.vector.partial.reduce
+; CHECK: call i32 @llvm.vector.reduce.add
+;
+; CHECK-FIXED-LABEL: define i32 @partial_reduce_bad_passthrough(
+; CHECK-FIXED-NOT: llvm.vector.partial.reduce
+; CHECK-FIXED: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.1 = add nsw i32 %sum, %bconv
+
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum.1, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.2 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
>From f28bcec2dc8679444682bfb398f3a3f0bdd4cc37 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 27 Jul 2026 00:04:56 -0700
Subject: [PATCH 07/13] Update tests after rebase.
---
.../VPlan/RISCV/reductions-evl.ll | 151 +++++++++---------
1 file changed, 75 insertions(+), 76 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index 5b1b19fffd7ec..d7b53c8da1d78 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -14,15 +14,15 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP2]]>, ir<%red.next>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
@@ -34,7 +34,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -90,15 +90,15 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector fast ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP2]]>, ir<%red.next>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
@@ -110,7 +110,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -168,15 +168,15 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector reassoc nnan ninf nsz ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector reassoc nnan ninf nsz ir<0.000000e+00>, ir<0.000000e+00>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi (fmuladd) reassoc nnan ninf nsz vp<[[VP3]]>, ir<%muladd>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi (fmuladd) reassoc nnan ninf nsz vp<[[VP2]]>, ir<%muladd>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
@@ -192,7 +192,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -255,7 +255,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
@@ -269,12 +269,11 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>, ir<1>
; CHECK-NEXT: WIDEN ir<%l.a> = vp.load vp<[[VP7]]>, vp<%evl>
; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%l.a>, ir<%start>
-; CHECK-NEXT: WIDEN ir<%cond> = select ir<%cmp2>, ir<%iv>, ir<%rdx>
-; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<true>, ir<%cond>, ir<%rdx>, vp<%evl>)
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<%cmp2>, ir<%iv>, ir<%rdx>, vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP9]]>, vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP2]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -334,15 +333,15 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
@@ -352,7 +351,7 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = vp<%evl> + reduce.add (ir<%load0> zext to i64, vp<%evl>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -411,15 +410,15 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, ir<%rdx.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, ir<%rdx.next>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
@@ -433,7 +432,7 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%mul>, vp<%evl>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -496,15 +495,15 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP9:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9:%[0-9]+]]>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
@@ -520,7 +519,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = vp<%evl> + reduce.add (ir<%mul> sext to i64, vp<%evl>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -589,15 +588,15 @@ define i32 @print_mulacc_negated(ptr %a, ptr %b) {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, ir<%add>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP2]]>, ir<%add>
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<1024>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
@@ -616,7 +615,7 @@ define i32 @print_mulacc_negated(ptr %a, ptr %b) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP10]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -678,23 +677,23 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64))
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
-; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
@@ -707,7 +706,7 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -759,23 +758,23 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64))
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
-; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
@@ -788,7 +787,7 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -841,23 +840,23 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64))
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, ir<%red.next>
-; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
@@ -870,7 +869,7 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -923,23 +922,23 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64))
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, vp<[[VP10:%[0-9]+]]>
-; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, vp<[[VP10:%[0-9]+]]>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
@@ -952,7 +951,7 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP11]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -1007,23 +1006,23 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64))
+; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + (-1 * (ptrtoaddr ptr %start to i64)) + (ptrtoaddr ptr %end to i64))
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP3]]>, vp<[[VP11:%[0-9]+]]>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
@@ -1036,7 +1035,7 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP12:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP12]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP12]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -1095,31 +1094,31 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + %n)
+; CHECK-NEXT: EMIT vp<[[VP2]]> = EXPAND SCEV (1 + %n)
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<1>
; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext ir<%b> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%conv>, ir<%conv>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP6:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, vp<[[VP7:%[0-9]+]]>
-; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP2]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EXPRESSION vp<[[VP7]]> = vp<%evl> + reduce.add (ir<%mul> zext to i64, vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP8]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
@@ -1190,7 +1189,7 @@ define i32 @print_umax_reduction(ptr %y) {
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP4:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
@@ -1207,7 +1206,7 @@ define i32 @print_umax_reduction(ptr %y) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP9]]>, vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP9]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP2]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
>From 9ca8c60ec32a4a0413eb0710749c591ea79764a1 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 27 Jul 2026 04:32:38 -0700
Subject: [PATCH 08/13] Address comments. Using decompose.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 12 ++++----
.../Vectorize/VPlanEVLTailFolding.cpp | 20 ++++++++++++-
.../Transforms/Vectorize/VPlanLowering.cpp | 2 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 29 ++-----------------
4 files changed, 28 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 34c1deaba597c..3048426843e89 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3668,13 +3668,13 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
}
- /// Clone this recipe and replace the reduction recipe to EVL recipe.
- VPExpressionRecipe *cloneWithEVL(VPValue *Mask, VPValue *EVL);
+ /// Return and insert the recipes of the expression back into the VPlan,
+ /// directly before the current recipe. Leaves the expression recipe empty,
+ /// which must be removed before codegen.
+ SmallVector<VPSingleDefRecipe *> decompose();
- /// Insert the recipes of the expression back into the VPlan,
- /// directly before the current recipe. Leaves the expression
- /// recipe empty, which must be removed before codegen.
- void decompose();
+ /// Returns the expression type of this recipe.
+ ExpressionTypes getExpressionType() const { return ExpressionType; }
unsigned getVFScaleFactor() const {
auto *PR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index fff6457137fea..478083406f157 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -241,7 +241,25 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
if (auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe))
if (match(Expr->getOperand(Expr->getNumOperands() - 1),
m_RemoveMask(HeaderMask, Mask))) {
- auto *NewExpr = Expr->cloneWithEVL(Mask, &EVL);
+ // Decompose first and construct with EVL recipes later.
+ SmallVector<VPSingleDefRecipe *> ExpressionRecipes(Expr->decompose());
+
+ // Convert recipes to EVL recipes.
+ for (auto [Idx, ExprR] : enumerate(ExpressionRecipes))
+ if (auto *EVLR = cast_if_present<VPSingleDefRecipe>(
+ optimizeMaskToEVL(HeaderMask, *ExprR, EVL))) {
+ VPSingleDefRecipe *OrigR = ExpressionRecipes[Idx];
+ EVLR->insertBefore(OrigR);
+ OrigR->replaceAllUsesWith(EVLR);
+ ExpressionRecipes[Idx] = EVLR;
+ OrigR->eraseFromParent();
+ }
+
+ auto *NewExpr =
+ new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes);
+
+ ExpressionRecipes[ExpressionRecipes.size() - 1]->replaceAllUsesWith(
+ NewExpr);
return NewExpr;
}
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
index d78a6e8c68681..6dc0390ad9097 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanLowering.cpp
@@ -598,7 +598,7 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
}
if (auto *Expr = dyn_cast<VPExpressionRecipe>(&R)) {
- Expr->decompose();
+ (void)Expr->decompose();
Expr->eraseFromParent();
continue;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5519d48893ead..ce9ad08923b06 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3580,7 +3580,7 @@ VPExpressionRecipe::VPExpressionRecipe(
R->replaceUsesOfWith(LiveIn, Tmp);
}
-void VPExpressionRecipe::decompose() {
+SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
for (auto *R : ExpressionRecipes)
// Since the list could contain duplicates, make sure the recipe hasn't
// already been inserted.
@@ -3591,32 +3591,7 @@ void VPExpressionRecipe::decompose() {
LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
replaceAllUsesWith(ExpressionRecipes.back());
- ExpressionRecipes.clear();
-}
-
-VPExpressionRecipe *VPExpressionRecipe::cloneWithEVL(VPValue *Mask,
- VPValue *EVL) {
- assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
- SmallVector<VPSingleDefRecipe *> NewExpressiondRecipes;
- for (auto *R : ExpressionRecipes)
- NewExpressiondRecipes.push_back(R->clone());
-
- for (auto *New : NewExpressiondRecipes) {
- for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
- New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
- // Update placeholder operands in the cloned recipe to use the external
- // operands, to be internalized when the cloned expression is constructed.
- for (const auto &[Placeholder, OutsideOp] :
- zip(LiveInPlaceholders, operands()))
- New->replaceUsesOfWith(Placeholder, OutsideOp);
- }
-
- // Convert to VPReductionEVLRecipe
- auto *Red = cast<VPReductionRecipe>(NewExpressiondRecipes.pop_back_val());
- auto *NewRed = new VPReductionEVLRecipe(*Red, *EVL, Mask);
- delete Red;
- NewExpressiondRecipes.push_back(NewRed);
- return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
+ return std::move(ExpressionRecipes);
}
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
>From d98113da16c1c5d9c2b4f8afb552b9f707c62d68 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 10 Aug 2026 23:37:31 -0700
Subject: [PATCH 09/13] Update tests after rebase.
---
.../RISCV/partial-reduce-with-predicate.ll | 12 +++----
.../VPlan/RISCV/partial-reduce-dot-product.ll | 12 +++----
.../VPlan/RISCV/reductions-evl.ll | 36 +++++++++----------
3 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
index 80c6ebe61c268..2b3cf2cd7da5c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-with-predicate.ll
@@ -40,7 +40,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
@@ -139,7 +139,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
@@ -241,7 +241,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
@@ -356,7 +356,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
@@ -472,7 +472,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
@@ -587,7 +587,7 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-FIXED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
; CHECK-FIXED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-FIXED: [[VECTOR_PH]]:
-; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; CHECK-FIXED-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 15
; CHECK-FIXED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-FIXED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-FIXED: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
index 210e38c53e965..7e810f89fd713 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -12,30 +12,30 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, vp<[[VP10:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP2]]>, vp<[[VP10:%[0-9]+]]> (VF scaled by 1/4)
; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<1024>, vector.ph ], [ vp<%avl.next>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.a>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i8, ir<%gep.a>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.a> = vp.load vp<[[VP8]]>, vp<%evl>
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, ir<%gep.b>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<%accum> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> sext to i32), vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP11:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add nuw vp<[[VP11]]>, vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index d7b53c8da1d78..c31a0338986b6 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -28,7 +28,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP8]]>, vp<%evl>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast vp.reduce.fadd (ir<%lv>, vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
@@ -104,7 +104,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP8]]>, vp<%evl>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast vp.reduce.fadd (ir<%lv>, vp<%evl>)
; CHECK-NEXT: EMIT-SCALAR vp<[[VP9:%[0-9]+]]> = zext vp<%evl> to i64
@@ -182,10 +182,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%l.a> = vp.load vp<[[VP8]]>, vp<%evl>
; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds float, ir<%arrayidx2>, ir<1>
; CHECK-NEXT: WIDEN ir<%l.b> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = fmul reassoc nnan ninf nsz ir<%l.a>, ir<%l.b>
; CHECK-NEXT: REDUCE ir<%muladd> = ir<%sum.07> + reassoc nnan ninf nsz vp.reduce.fadd (vp<[[VP10]]>, vp<%evl>)
@@ -266,7 +266,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP5]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>, ir<1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i64, ir<%gep.a>, ir<1>
; CHECK-NEXT: WIDEN ir<%l.a> = vp.load vp<[[VP7]]>, vp<%evl>
; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%l.a>, ir<%start>
; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<%cmp2>, ir<%iv>, ir<%rdx>, vp<%evl>)
@@ -346,7 +346,7 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = vp<%evl> + reduce.add (ir<%load0> zext to i64, vp<%evl>)
; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
@@ -423,10 +423,10 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i64, ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i64, ir<%arrayidx1>, ir<1>
; CHECK-NEXT: WIDEN ir<%load1> = vp.load vp<[[VP8]]>, vp<%evl>
; CHECK-NEXT: WIDEN ir<%mul> = mul nsw ir<%load0>, ir<%load1>
; CHECK-NEXT: REDUCE ir<%rdx.next> = ir<%rdx> + vp.reduce.add (ir<%mul>, vp<%evl>)
@@ -508,10 +508,10 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<%evl>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i16, ir<%arrayidx>, ir<1>
; CHECK-NEXT: WIDEN ir<%load0> = vp.load vp<[[VP7]]>, vp<%evl>
; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i16, ir<%arrayidx1>, ir<1>
; CHECK-NEXT: WIDEN ir<%load1> = vp.load vp<[[VP8]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%conv0> = sext ir<%load0> to i32
; CHECK-NEXT: WIDEN-CAST ir<%conv1> = sext ir<%load1> to i32
@@ -602,11 +602,11 @@ define i32 @print_mulacc_negated(ptr %a, ptr %b) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP6]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.a>, ir<1>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer i8, ir<%gep.a>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.a> = vp.load vp<[[VP8]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[VP7]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.b>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, ir<%gep.b>, ir<1>
; CHECK-NEXT: WIDEN ir<%load.b> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>
@@ -698,7 +698,7 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63>
@@ -779,7 +779,7 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<63>, ir<%l.ext>
@@ -861,7 +861,7 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
@@ -943,7 +943,7 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%l.ext>, ir<63>
@@ -1027,7 +1027,7 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP7:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP7]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP8]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer i8, vp<%next.gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%l> = vp.load vp<[[VP9]]>, vp<%evl>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
@@ -1199,7 +1199,7 @@ define i32 @print_umax_reduction(ptr %y) {
; CHECK-NEXT: EMIT-SCALAR vp<[[VP5:%[0-9]+]]> = zext vp<%evl> to i64
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP5]]>
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%gep>, ir<1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds i32, ir<%gep>, ir<1>
; CHECK-NEXT: WIDEN ir<%lv> = vp.load vp<[[VP7]]>, vp<%evl>
; CHECK-NEXT: WIDEN-INTRINSIC ir<%red.next> = call llvm.umax(ir<%lv>, ir<%red>)
; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8]]> = call llvm.vp.merge(ir<true>, ir<%red.next>, ir<%red>, vp<%evl>)
>From 411edb2223ad255059177e4a3fd8e0947e98fbbc Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 13 Aug 2026 18:30:35 -0700
Subject: [PATCH 10/13] Address comments.
---
.../Vectorize/VPlanEVLTailFolding.cpp | 8 +-
.../VPlan/RISCV/partial-reduce-dot-product.ll | 25 +-
.../VPlan/RISCV/reductions-evl.ll | 304 +-----------------
3 files changed, 5 insertions(+), 332 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 478083406f157..8d09bb0a1add7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -248,16 +248,14 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
for (auto [Idx, ExprR] : enumerate(ExpressionRecipes))
if (auto *EVLR = cast_if_present<VPSingleDefRecipe>(
optimizeMaskToEVL(HeaderMask, *ExprR, EVL))) {
- VPSingleDefRecipe *OrigR = ExpressionRecipes[Idx];
- EVLR->insertBefore(OrigR);
- OrigR->replaceAllUsesWith(EVLR);
+ EVLR->insertBefore(ExprR);
+ ExprR->replaceAllUsesWith(EVLR);
+ ExprR->eraseFromParent();
ExpressionRecipes[Idx] = EVLR;
- OrigR->eraseFromParent();
}
auto *NewExpr =
new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes);
-
ExpressionRecipes[ExpressionRecipes.size() - 1]->replaceAllUsesWith(
NewExpr);
return NewExpr;
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
index 7e810f89fd713..30781b88f44e0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "ir-bb<for.exit>" --version 6
; RUN: opt -mtriple riscv64 -passes=loop-vectorize -force-vector-width=4 -scalable-vectorization=on -mattr=+v,+experimental-zvdot4a8i -disable-output 2>&1 -vplan-print-after=optimizeEVLMasks < %s | FileCheck %s
define i32 @vdota4(ptr %a, ptr %b) #0 {
@@ -44,29 +44,6 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP10]]>
; CHECK-NEXT: Successor(s): ir-bb<for.exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.exit>:
-; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[VP13]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<for.body>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.body>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %for.body ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv
-; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1
-; CHECK-NEXT: IR %ext.a = sext i8 %load.a to i32
-; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv
-; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1
-; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32
-; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a
-; CHECK-NEXT: IR %add = add i32 %mul, %accum
-; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
-; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
index c31a0338986b6..2daf7496a618e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/reductions-evl.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "ir-bb<exit>" --version 6
; RUN: opt -passes=loop-vectorize -vplan-print-after="optimizeEVLMasks" -mtriple riscv64 -mattr=+v,+experimental-zvdot4a8i -force-vector-width=4 -scalable-vectorization=on -prefer-inloop-reductions -disable-output %s 2>&1 | FileCheck %s
; Tests for printing VPlans with reductions.
@@ -43,24 +43,6 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %red = phi float [ %red.next, %loop ], [ 0.000000e+00, %entry ] (extra operand: ir<0.000000e+00> from scalar.ph)
-; CHECK-NEXT: IR %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv
-; CHECK-NEXT: IR %lv = load float, ptr %arrayidx, align 4
-; CHECK-NEXT: IR %red.next = fadd fast float %lv, %red
-; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
-; CHECK-NEXT: IR %exitcond = icmp eq i64 %iv.next, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -120,24 +102,6 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
; CHECK-NEXT: CLONE store vp<[[VP11]]>, ir<%dst>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %red = phi float [ %red.next, %loop ], [ 0.000000e+00, %entry ] (extra operand: ir<0.000000e+00> from scalar.ph)
-; CHECK-NEXT: IR %arrayidx = getelementptr inbounds float, ptr %y, i64 %iv
-; CHECK-NEXT: IR %lv = load float, ptr %arrayidx, align 4
-; CHECK-NEXT: IR %red.next = fadd fast float %lv, %red
-; CHECK-NEXT: IR store float %red.next, ptr %dst, align 4
-; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
-; CHECK-NEXT: IR %exitcond = icmp eq i64 %iv.next, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -201,26 +165,6 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (fmuladd, in-loop) reassoc nnan ninf nsz ir<%muladd>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %loop ] (extra operand: ir<0.000000e+00> from scalar.ph)
-; CHECK-NEXT: IR %arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
-; CHECK-NEXT: IR %l.a = load float, ptr %arrayidx, align 4
-; CHECK-NEXT: IR %arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv
-; CHECK-NEXT: IR %l.b = load float, ptr %arrayidx2, align 4
-; CHECK-NEXT: IR %muladd = tail call reassoc nnan ninf nsz float @llvm.fmuladd.f32(float %l.a, float %l.b, float %sum.07)
-; CHECK-NEXT: IR %iv.next = add nuw nsw i64 %iv, 1
-; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -284,25 +228,6 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ne vp<[[VP11]]>, ir<-9223372036854775808>
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = select vp<[[VP12]]>, vp<[[VP11]]>, ir<%start>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %inc, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %rdx = phi i64 [ %start, %entry ], [ %cond, %loop ] (extra operand: ir<%start> from scalar.ph)
-; CHECK-NEXT: IR %gep.a = getelementptr inbounds i64, ptr %a, i64 %iv
-; CHECK-NEXT: IR %l.a = load i64, ptr %gep.a, align 8
-; CHECK-NEXT: IR %cmp2 = icmp eq i64 %l.a, %start
-; CHECK-NEXT: IR %cond = select i1 %cmp2, i64 %iv, i64 %rdx
-; CHECK-NEXT: IR %inc = add nuw nsw i64 %iv, 1
-; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %inc, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -360,25 +285,6 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP8]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %arrayidx = getelementptr inbounds i32, ptr %x, i32 %iv
-; CHECK-NEXT: IR %load0 = load i32, ptr %arrayidx, align 4
-; CHECK-NEXT: IR %conv0 = zext i32 %load0 to i64
-; CHECK-NEXT: IR %rdx.next = add nsw i64 %rdx, %conv0
-; CHECK-NEXT: IR %iv.next = add nuw nsw i32 %iv, 1
-; CHECK-NEXT: IR %exitcond = icmp eq i32 %iv.next, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -441,27 +347,6 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%rdx.next>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %arrayidx = getelementptr inbounds i64, ptr %x, i32 %iv
-; CHECK-NEXT: IR %load0 = load i64, ptr %arrayidx, align 8
-; CHECK-NEXT: IR %arrayidx1 = getelementptr inbounds i64, ptr %y, i32 %iv
-; CHECK-NEXT: IR %load1 = load i64, ptr %arrayidx1, align 8
-; CHECK-NEXT: IR %mul = mul nsw i64 %load0, %load1
-; CHECK-NEXT: IR %rdx.next = add nsw i64 %rdx, %mul
-; CHECK-NEXT: IR %iv.next = add nuw nsw i32 %iv, 1
-; CHECK-NEXT: IR %exitcond = icmp eq i32 %iv.next, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -528,30 +413,6 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP9]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %arrayidx = getelementptr inbounds i16, ptr %x, i32 %iv
-; CHECK-NEXT: IR %load0 = load i16, ptr %arrayidx, align 8
-; CHECK-NEXT: IR %arrayidx1 = getelementptr inbounds i16, ptr %y, i32 %iv
-; CHECK-NEXT: IR %load1 = load i16, ptr %arrayidx1, align 8
-; CHECK-NEXT: IR %conv0 = sext i16 %load0 to i32
-; CHECK-NEXT: IR %conv1 = sext i16 %load1 to i32
-; CHECK-NEXT: IR %mul = mul nsw i32 %conv0, %conv1
-; CHECK-NEXT: IR %conv = sext i32 %mul to i64
-; CHECK-NEXT: IR %rdx.next = add nsw i64 %rdx, %conv
-; CHECK-NEXT: IR %iv.next = add nuw nsw i32 %iv, 1
-; CHECK-NEXT: IR %exitcond = icmp eq i32 %iv.next, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -624,30 +485,6 @@ define i32 @print_mulacc_negated(ptr %a, ptr %b) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%add>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %add.lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %accum = phi i32 [ 0, %entry ], [ %add, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %gep.a = getelementptr i8, ptr %a, i64 %iv
-; CHECK-NEXT: IR %load.a = load i8, ptr %gep.a, align 1
-; CHECK-NEXT: IR %ext.a = zext i8 %load.a to i32
-; CHECK-NEXT: IR %gep.b = getelementptr i8, ptr %b, i64 %iv
-; CHECK-NEXT: IR %load.b = load i8, ptr %gep.b, align 1
-; CHECK-NEXT: IR %ext.b = zext i8 %load.b to i32
-; CHECK-NEXT: IR %mul = mul i32 %ext.b, %ext.a
-; CHECK-NEXT: IR %sub = sub i32 0, %mul
-; CHECK-NEXT: IR %add = add i32 %accum, %sub
-; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
-; CHECK-NEXT: IR %exitcond.not = icmp eq i64 %iv.next, 1024
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -715,25 +552,6 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
-; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
-; CHECK-NEXT: IR %l.ext = zext i8 %l to i32
-; CHECK-NEXT: IR %mul = mul i32 %l.ext, 63
-; CHECK-NEXT: IR %red.next = add i32 %red, %mul
-; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
-; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -796,25 +614,6 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
-; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
-; CHECK-NEXT: IR %l.ext = zext i8 %l to i32
-; CHECK-NEXT: IR %mul = mul i32 63, %l.ext
-; CHECK-NEXT: IR %red.next = add i32 %red, %mul
-; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
-; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -878,25 +677,6 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) ir<%red.next>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
-; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
-; CHECK-NEXT: IR %l.ext = sext i8 %l to i32
-; CHECK-NEXT: IR %mul = mul i32 %l.ext, 128
-; CHECK-NEXT: IR %red.next = add i32 %red, %mul
-; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
-; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -960,26 +740,6 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP10]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
-; CHECK-NEXT: IR %red = phi i64 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
-; CHECK-NEXT: IR %l.ext = zext i8 %l to i32
-; CHECK-NEXT: IR %mul = mul i32 %l.ext, 63
-; CHECK-NEXT: IR %mul.ext = zext i32 %mul to i64
-; CHECK-NEXT: IR %red.next = add i64 %red, %mul.ext
-; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
-; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -1044,26 +804,6 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %gep.iv.next, %loop ] (extra operand: ir<%start> from scalar.ph)
-; CHECK-NEXT: IR %red = phi i64 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1
-; CHECK-NEXT: IR %l.ext = sext i8 %l to i32
-; CHECK-NEXT: IR %mul = mul i32 %l.ext, 128
-; CHECK-NEXT: IR %mul.ext = sext i32 %mul to i64
-; CHECK-NEXT: IR %red.next = add i64 %red, %mul.ext
-; CHECK-NEXT: IR %gep.iv.next = getelementptr i8, ptr %ptr.iv, i64 1
-; CHECK-NEXT: IR %ec = icmp eq ptr %ptr.iv, %end
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -1127,30 +867,6 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP7]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %add.lcssa = phi i64 [ %add, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %res1 = phi i64 [ 0, %entry ], [ %load.ext.ext, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %res2 = phi i64 [ 0, %entry ], [ %add, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %load = load i16, ptr %a, align 2
-; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
-; CHECK-NEXT: IR %conv = sext i16 %b to i32
-; CHECK-NEXT: IR %mul = mul i32 %conv, %conv
-; CHECK-NEXT: IR %mul.ext = zext i32 %mul to i64
-; CHECK-NEXT: IR %add = add i64 %res2, %mul.ext
-; CHECK-NEXT: IR %second_use = or i32 %mul, %c
-; CHECK-NEXT: IR %load.ext = sext i16 %load to i32
-; CHECK-NEXT: IR %load.ext.ext = sext i32 %load.ext to i64
-; CHECK-NEXT: IR %exitcond740.not = icmp eq i64 %iv, %n
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
@@ -1215,24 +931,6 @@ define i32 @print_umax_reduction(ptr %y) {
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (umax) vp<[[VP8]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
-; CHECK-NEXT: No successors
-; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<loop>
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<loop>:
-; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %red = phi i32 [ 0, %entry ], [ %red.next, %loop ] (extra operand: ir<0> from scalar.ph)
-; CHECK-NEXT: IR %gep = getelementptr inbounds i32, ptr %y, i64 %iv
-; CHECK-NEXT: IR %lv = load i32, ptr %gep, align 4
-; CHECK-NEXT: IR %red.next = call i32 @llvm.umax.i32(i32 %lv, i32 %red)
-; CHECK-NEXT: IR %iv.next = add i64 %iv, 1
-; CHECK-NEXT: IR %ec = icmp eq i64 %iv.next, 100
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
;
entry:
br label %loop
>From a822aee228ce86d0ed93f21a4f9d499697add418 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 18 Aug 2026 22:45:45 -0700
Subject: [PATCH 11/13] Address comments.
---
.../Vectorize/VPlanEVLTailFolding.cpp | 60 ++++++++++++-------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 15 +++--
2 files changed, 47 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index 8d09bb0a1add7..b3fc846aa8bc6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -237,32 +237,43 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
Mask ? Mask : Plan->getTrue(), &EVL},
IntrR->getScalarType(), {}, {}, DL);
- // Transforms the VPReductionRecipe inside the VPExpressionRecipe.
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe))
- if (match(Expr->getOperand(Expr->getNumOperands() - 1),
- m_RemoveMask(HeaderMask, Mask))) {
- // Decompose first and construct with EVL recipes later.
- SmallVector<VPSingleDefRecipe *> ExpressionRecipes(Expr->decompose());
-
- // Convert recipes to EVL recipes.
- for (auto [Idx, ExprR] : enumerate(ExpressionRecipes))
- if (auto *EVLR = cast_if_present<VPSingleDefRecipe>(
- optimizeMaskToEVL(HeaderMask, *ExprR, EVL))) {
- EVLR->insertBefore(ExprR);
- ExprR->replaceAllUsesWith(EVLR);
- ExprR->eraseFromParent();
- ExpressionRecipes[Idx] = EVLR;
- }
-
- auto *NewExpr =
- new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes);
- ExpressionRecipes[ExpressionRecipes.size() - 1]->replaceAllUsesWith(
- NewExpr);
- return NewExpr;
- }
return nullptr;
}
+// Decompose the expression recipe and transform each contained recipe into
+// an EVL recipe.
+static bool optimizeExpressionRecipeToEVL(VPValue *HeaderMask,
+ VPRecipeBase &CurRecipe,
+ VPValue &EVL) {
+
+ auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe);
+ if (!Expr)
+ return false;
+
+ // Decompose first and construct with EVL recipes later.
+ SmallVector<VPSingleDefRecipe *> ExpressionRecipes(Expr->decompose());
+
+ // Convert recipes to EVL recipes.
+ for (auto [Idx, R] : enumerate(ExpressionRecipes))
+ if (auto *EVLR = cast_if_present<VPSingleDefRecipe>(
+ optimizeMaskToEVL(HeaderMask, *R, EVL))) {
+ EVLR->insertBefore(R);
+ R->replaceAllUsesWith(EVLR);
+ R->eraseFromParent();
+ ExpressionRecipes[Idx] = EVLR;
+ }
+
+ auto *NewExpr =
+ new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes);
+ ExpressionRecipes.back()->replaceAllUsesWith(NewExpr);
+ NewExpr->insertBefore(Expr);
+ Expr->eraseFromParent();
+ // Need to delete HeaderMask explicitly if the expression recipe is its only
+ // user.
+ vputils::recursivelyDeleteDeadRecipes(HeaderMask);
+ return true;
+}
+
/// Optimize away any EVL-based header masks to VP intrinsic based recipes.
/// The transforms here need to preserve the original semantics.
void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
@@ -282,6 +293,9 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
SmallVector<VPRecipeBase *> OldRecipes;
for (VPUser *U : vputils::collectUsersRecursively(HeaderMask)) {
VPRecipeBase *R = cast<VPRecipeBase>(U);
+ // Transform recipes contained by an expression recipe into EVL recipes.
+ if (optimizeExpressionRecipeToEVL(HeaderMask, *R, *EVL))
+ continue;
if (auto *NewR = optimizeMaskToEVL(HeaderMask, *R, *EVL)) {
NewR->insertBefore(R);
for (auto [Old, New] :
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ce9ad08923b06..bfcd1e43f5172 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3443,8 +3443,8 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
Value *NewRed;
if (isPartialReduction()) {
- // For partial reduction, we need to generate the predicated select
- // (vp.merge) since `@llvm.vector.partial.reduce()` doesn't have vector
+ // For partial reductions, we need to generate a predicated select
+ // (vp.merge) since `@llvm.vector.partial.reduce()` doesn't have a vector
// predicated version.
VectorType *VecTy = cast<VectorType>(VecOp->getType());
Value *Identity = getRecurrenceIdentity(Kind, VecTy->getElementType(),
@@ -3581,17 +3581,22 @@ VPExpressionRecipe::VPExpressionRecipe(
}
SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
- for (auto *R : ExpressionRecipes)
+ SmallVector<VPSingleDefRecipe *> InsertedRecipes;
+ for (auto *R : ExpressionRecipes) {
// Since the list could contain duplicates, make sure the recipe hasn't
// already been inserted.
- if (!R->getParent())
+ if (!R->getParent()) {
R->insertBefore(this);
+ InsertedRecipes.push_back(R);
+ }
+ }
for (const auto &[Idx, Op] : enumerate(operands()))
LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
replaceAllUsesWith(ExpressionRecipes.back());
- return std::move(ExpressionRecipes);
+ ExpressionRecipes.clear();
+ return InsertedRecipes;
}
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
>From cccc82a0ecefd74d890857c65c6ed84be39a58bf Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Thu, 20 Aug 2026 23:46:51 -0700
Subject: [PATCH 12/13] Address comments.
---
.../Transforms/Vectorize/VPlanEVLTailFolding.cpp | 16 +++++++---------
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +++-----
.../VPlan/RISCV/partial-reduce-dot-product.ll | 4 ++--
3 files changed, 12 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index b3fc846aa8bc6..f3fcb11a166b2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -242,9 +242,10 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
// Decompose the expression recipe and transform each contained recipe into
// an EVL recipe.
-static bool optimizeExpressionRecipeToEVL(VPValue *HeaderMask,
- VPRecipeBase &CurRecipe,
- VPValue &EVL) {
+static bool
+optimizeExpressionRecipeToEVL(VPValue *HeaderMask, VPRecipeBase &CurRecipe,
+ VPValue &EVL,
+ SmallVector<VPRecipeBase *> &OldRecipes) {
auto *Expr = dyn_cast<VPExpressionRecipe>(&CurRecipe);
if (!Expr)
@@ -259,7 +260,7 @@ static bool optimizeExpressionRecipeToEVL(VPValue *HeaderMask,
optimizeMaskToEVL(HeaderMask, *R, EVL))) {
EVLR->insertBefore(R);
R->replaceAllUsesWith(EVLR);
- R->eraseFromParent();
+ OldRecipes.push_back(R);
ExpressionRecipes[Idx] = EVLR;
}
@@ -267,10 +268,7 @@ static bool optimizeExpressionRecipeToEVL(VPValue *HeaderMask,
new VPExpressionRecipe(Expr->getExpressionType(), ExpressionRecipes);
ExpressionRecipes.back()->replaceAllUsesWith(NewExpr);
NewExpr->insertBefore(Expr);
- Expr->eraseFromParent();
- // Need to delete HeaderMask explicitly if the expression recipe is its only
- // user.
- vputils::recursivelyDeleteDeadRecipes(HeaderMask);
+ OldRecipes.push_back(Expr);
return true;
}
@@ -294,7 +292,7 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
for (VPUser *U : vputils::collectUsersRecursively(HeaderMask)) {
VPRecipeBase *R = cast<VPRecipeBase>(U);
// Transform recipes contained by an expression recipe into EVL recipes.
- if (optimizeExpressionRecipeToEVL(HeaderMask, *R, *EVL))
+ if (optimizeExpressionRecipeToEVL(HeaderMask, *R, *EVL, OldRecipes))
continue;
if (auto *NewR = optimizeMaskToEVL(HeaderMask, *R, *EVL)) {
NewR->insertBefore(R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bfcd1e43f5172..575f142bfab48 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3581,22 +3581,20 @@ VPExpressionRecipe::VPExpressionRecipe(
}
SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
- SmallVector<VPSingleDefRecipe *> InsertedRecipes;
for (auto *R : ExpressionRecipes) {
// Since the list could contain duplicates, make sure the recipe hasn't
// already been inserted.
- if (!R->getParent()) {
+ if (!R->getParent())
R->insertBefore(this);
- InsertedRecipes.push_back(R);
- }
}
for (const auto &[Idx, Op] : enumerate(operands()))
LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
replaceAllUsesWith(ExpressionRecipes.back());
+ SmallVector<VPSingleDefRecipe *> DecomposedRecipes(ExpressionRecipes);
ExpressionRecipes.clear();
- return InsertedRecipes;
+ return DecomposedRecipes;
}
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
index 30781b88f44e0..e971ff766b053 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/partial-reduce-dot-product.ll
@@ -48,7 +48,7 @@ define i32 @vdota4(ptr %a, ptr %b) #0 {
entry:
br label %for.body
-for.body: ; preds = %for.body, %entry
+for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
%gep.a = getelementptr i8, ptr %a, i64 %iv
@@ -63,6 +63,6 @@ for.body: ; preds = %for.body, %entry
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %for.exit, label %for.body
-for.exit: ; preds = %for.body
+for.exit:
ret i32 %add
}
>From 065bef87efe796d01848744ca2dc69d5bb6c30f8 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Sun, 23 Aug 2026 19:52:53 -0700
Subject: [PATCH 13/13] Using SmallSetVector to prevent duplicate recipes
issue.
---
llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp | 7 +++++--
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index f3fcb11a166b2..5fe924ea61374 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -19,6 +19,7 @@
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/Intrinsics.h"
@@ -253,15 +254,17 @@ optimizeExpressionRecipeToEVL(VPValue *HeaderMask, VPRecipeBase &CurRecipe,
// Decompose first and construct with EVL recipes later.
SmallVector<VPSingleDefRecipe *> ExpressionRecipes(Expr->decompose());
+ SmallSetVector<VPSingleDefRecipe *, 4> UniqueExpressionRecipes(
+ ExpressionRecipes.begin(), ExpressionRecipes.end());
// Convert recipes to EVL recipes.
- for (auto [Idx, R] : enumerate(ExpressionRecipes))
+ for (auto *R : UniqueExpressionRecipes)
if (auto *EVLR = cast_if_present<VPSingleDefRecipe>(
optimizeMaskToEVL(HeaderMask, *R, EVL))) {
EVLR->insertBefore(R);
R->replaceAllUsesWith(EVLR);
OldRecipes.push_back(R);
- ExpressionRecipes[Idx] = EVLR;
+ replace(ExpressionRecipes, R, EVLR);
}
auto *NewExpr =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 575f142bfab48..169a96ff12fda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3581,12 +3581,11 @@ VPExpressionRecipe::VPExpressionRecipe(
}
SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
- for (auto *R : ExpressionRecipes) {
+ for (auto *R : ExpressionRecipes)
// Since the list could contain duplicates, make sure the recipe hasn't
// already been inserted.
if (!R->getParent())
R->insertBefore(this);
- }
for (const auto &[Idx, Op] : enumerate(operands()))
LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
More information about the llvm-commits
mailing list