[llvm] [VPlan] Print RecurKind when dumping VPReductionPHI (PR #190114)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 2 08:47:54 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/190114
>From 07bdcca346a8593e39feabe66e9c76ca076a745c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 14:05:02 +0800
Subject: [PATCH 1/2] [VPlan] Print RecurKind when dumping VPReductionPHI
This makes it easier to see what type of recurrence is in a plan when viewing the debug output.
It also adds an explicit string for each recurrence kind instead of using through the opcode, as there were some kinds that share opcodes with other kinds like fmuladd that we were forgetting to handle.
---
llvm/include/llvm/Analysis/IVDescriptors.h | 84 +++++++++++++++++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 47 +----------
.../VPlan/AArch64/vplan-printing.ll | 8 +-
.../RISCV/vplan-vp-intrinsics-reduction.ll | 8 +-
.../conditional-scalar-assignment-vplan.ll | 8 +-
.../LoopVectorize/VPlan/find-last.ll | 2 +-
...-order-recurrence-sink-replicate-region.ll | 2 +-
.../LoopVectorize/VPlan/icmp-uniforms.ll | 2 +-
...ave-conditional-scalar-assignment-vplan.ll | 8 +-
.../LoopVectorize/VPlan/tail-folding.ll | 2 +-
.../VPlan/vplan-printing-reductions.ll | 40 ++++-----
.../LoopVectorize/X86/reduction-small-size.ll | 2 +-
12 files changed, 129 insertions(+), 84 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 8383da0f9e879..4948effe6fc78 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -458,6 +458,90 @@ class InductionDescriptor {
SmallVector<Instruction *, 2> RedundantCasts;
};
+inline raw_ostream &operator<<(raw_ostream &OS, const RecurKind &Kind) {
+ switch (Kind) {
+ case RecurKind::None:
+ OS << "none";
+ break;
+ case RecurKind::Add:
+ OS << "add";
+ break;
+ case RecurKind::Sub:
+ OS << "sub";
+ break;
+ case RecurKind::AddChainWithSubs:
+ OS << "add-chain-with-subs";
+ break;
+ case RecurKind::Mul:
+ OS << "mul";
+ break;
+ case RecurKind::Or:
+ OS << "or";
+ break;
+ case RecurKind::And:
+ OS << "and";
+ break;
+ case RecurKind::Xor:
+ OS << "xor";
+ break;
+ case RecurKind::SMin:
+ OS << "smin";
+ break;
+ case RecurKind::SMax:
+ OS << "smax";
+ break;
+ case RecurKind::UMin:
+ OS << "umin";
+ break;
+ case RecurKind::UMax:
+ OS << "umax";
+ break;
+ case RecurKind::FAdd:
+ OS << "fadd";
+ break;
+ case RecurKind::FMul:
+ OS << "fmul";
+ break;
+ case RecurKind::FMin:
+ OS << "fmin";
+ break;
+ case RecurKind::FMax:
+ OS << "fmax";
+ break;
+ case RecurKind::FMinNum:
+ OS << "fminnum";
+ break;
+ case RecurKind::FMaxNum:
+ OS << "fmaxnum";
+ break;
+ case RecurKind::FMinimum:
+ OS << "fminimum";
+ break;
+ case RecurKind::FMaximum:
+ OS << "fmaximum";
+ break;
+ case RecurKind::FMinimumNum:
+ OS << "fminimumnum";
+ break;
+ case RecurKind::FMaximumNum:
+ OS << "fmaximumnum";
+ break;
+ case RecurKind::FMulAdd:
+ OS << "fmuladd";
+ break;
+ case RecurKind::AnyOf:
+ OS << "any-of";
+ break;
+ case RecurKind::FindIV:
+ OS << "find-iv";
+ break;
+ case RecurKind::FindLast:
+ OS << "find-last";
+ break;
+ }
+ return OS;
+}
+
} // end namespace llvm
#endif // LLVM_ANALYSIS_IVDESCRIPTORS_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7eefd77045050..4ce6af4038889 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2247,49 +2247,8 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
O << " nneg";
break;
case OperationType::ReductionOp: {
- RecurKind RK = getRecurKind();
O << " (";
- switch (RK) {
- case RecurKind::AnyOf:
- O << "any-of";
- break;
- case RecurKind::FindLast:
- O << "find-last";
- break;
- case RecurKind::SMax:
- O << "smax";
- break;
- case RecurKind::SMin:
- O << "smin";
- break;
- case RecurKind::UMax:
- O << "umax";
- break;
- case RecurKind::UMin:
- O << "umin";
- break;
- case RecurKind::FMinNum:
- O << "fminnum";
- break;
- case RecurKind::FMaxNum:
- O << "fmaxnum";
- break;
- case RecurKind::FMinimum:
- O << "fminimum";
- break;
- case RecurKind::FMaximum:
- O << "fmaximum";
- break;
- case RecurKind::FMinimumNum:
- O << "fminimumnum";
- break;
- case RecurKind::FMaximumNum:
- O << "fmaximumnum";
- break;
- default:
- O << Instruction::getOpcodeName(RecurrenceDescriptor::getOpcode(RK));
- break;
- }
+ O << getRecurKind();
if (isReductionInLoop())
O << ", in-loop";
if (isReductionOrdered())
@@ -4670,7 +4629,9 @@ void VPReductionPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN-REDUCTION-PHI ";
printAsOperand(O, SlotTracker);
- O << " = phi";
+ O << " = phi (";
+ O << Kind;
+ O << ")";
printFlags(O);
printOperands(O, SlotTracker);
if (getVFScaleFactor() > 1)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
index 587531cda95c3..b5b1fbaca471d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing.ll
@@ -22,7 +22,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) "target-features"="+neon,+do
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi vp<[[RDX_START]]>, vp<[[REDUCE:%.+]]> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi (add) vp<[[RDX_START]]>, vp<[[REDUCE:%.+]]> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>
@@ -80,7 +80,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) "target-features"="+neon,+do
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT-SCALAR vp<[[EP_IV:%.+]]> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi (add) vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]> (VF scaled by 1/4)
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
; CHECK-NEXT: WIDEN ir<%load.a> = load ir<%gep.a>
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%index>
@@ -144,7 +144,7 @@ define i32 @print_partial_reduction_predication(ptr %a, ptr %b, i64 %N) "target-
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ACTIVE-LANE-MASK-PHI vp<[[MASK:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi vp<[[RDX_START]]>, vp<[[REDUCE:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[RDX_START]]>, vp<[[REDUCE:%[0-9]+]]> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[STEPS:%[0-9]+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[PTR_A:%[0-9]+]]> = vector-pointer ir<%gep.a>
@@ -208,7 +208,7 @@ define i32 @print_partial_reduction_ext_mul(ptr %a, ptr %b) "target-features"="+
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi vp<[[RDX_START]]>, vp<[[REDUCE:%.+]]> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi (add) vp<[[RDX_START]]>, vp<[[REDUCE:%.+]]> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
index de18dfbd605c0..1e0c94b816823 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -37,7 +37,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: vector.body:
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
; IF-EVL-OUTLOOP-NEXT: CURRENT-ITERATION-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
-; IF-EVL-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
+; IF-EVL-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi (add) vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
@@ -77,7 +77,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: vector.body:
; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
; IF-EVL-INLOOP-NEXT: CURRENT-ITERATION-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
-; IF-EVL-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; IF-EVL-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi (add) vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
@@ -116,7 +116,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-OUTLOOP-NEXT: <x1> vector loop: {
; NO-VP-OUTLOOP-NEXT: vector.body:
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; NO-VP-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; NO-VP-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi (add) vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
; NO-VP-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
; NO-VP-OUTLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; NO-VP-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
@@ -164,7 +164,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; NO-VP-INLOOP-NEXT: <x1> vector loop: {
; NO-VP-INLOOP-NEXT: vector.body:
; NO-VP-INLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; NO-VP-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; NO-VP-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi (add) vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
; NO-VP-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
; NO-VP-INLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; NO-VP-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
index cf47da0cae2ea..c3e09ce35ca0d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
@@ -32,7 +32,7 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<-1>, vp<[[VP9:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP9:%[0-9]+]]>
; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP8:%[0-9]+]]>, vector.body ]
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<[[VP5]]>
@@ -92,7 +92,7 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: <x1> vector loop: {
; CHECK-TF-NEXT: vector.body:
; CHECK-TF-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<-1>, vp<[[VP13:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP13:%[0-9]+]]>
; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP12:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
@@ -179,7 +179,7 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<%default_val>, vp<[[VP11:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP11:%[0-9]+]]>
; CHECK-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP10:%[0-9]+]]>, if.then.0 ]
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%a.addr> = getelementptr inbounds nuw ir<%a>, vp<[[VP5]]>
@@ -256,7 +256,7 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: <x1> vector loop: {
; CHECK-TF-NEXT: vector.body:
; CHECK-TF-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<%default_val>, vp<[[VP14:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP14:%[0-9]+]]>
; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP13:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll b/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
index 15beb9ef83842..818340f25fbeb 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
@@ -19,7 +19,7 @@ define i32 @find_last_with_select(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi ir<1>, ir<%sel>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-last) ir<1>, ir<%sel>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>
; CHECK-NEXT: WIDEN ir<%load.a> = load vp<[[VP4]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
index 08720f73ef0ef..f05b659919163 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
@@ -239,7 +239,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi vp<[[VP3]]>, ir<%and.red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi (and) vp<[[VP3]]>, ir<%and.red.next>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
index 5caf66a247e6d..984ac6e50c92a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/icmp-uniforms.ll
@@ -29,7 +29,7 @@ define i32 @more_than_one_use(ptr %a, i64 %n) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%i> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%r> = phi vp<[[VP4]]>, ir<%tmp3>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%r> = phi (add) vp<[[VP4]]>, ir<%tmp3>
; CHECK-NEXT: WIDEN ir<%i.next> = add nuw nsw ir<%i>, ir<1>
; CHECK-NEXT: WIDEN ir<%cond> = icmp slt ir<%i.next>, ir<%n>
; CHECK-NEXT: WIDEN ir<%tmp0> = select ir<%cond>, ir<%i.next>, ir<0>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
index a783674ac869c..d28f3a2b63fbc 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/interleave-conditional-scalar-assignment-vplan.ll
@@ -32,8 +32,8 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-EMPTY:
; IC2-NEXT: vector.body:
; IC2-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<-1>, vp<[[VP10:%[0-9]+]]>
-; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi ir<-1>, vp<[[VP11:%[0-9]+]]>
+; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP10:%[0-9]+]]>
+; IC2-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP11:%[0-9]+]]>
; IC2-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP8:%[0-9]+]]>, vector.body ]
; IC2-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP9:%[0-9]+]]>, vector.body ]
; IC2-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<%index>
@@ -97,8 +97,8 @@ define i32 @find_last_int_select(i64 %N, ptr %data, i32 %a) {
; IC2-TF-EMPTY:
; IC2-TF-NEXT: vector.body:
; IC2-TF-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
-; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi ir<-1>, vp<[[VP16:%[0-9]+]]>
-; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi ir<-1>, vp<[[VP17:%[0-9]+]]>
+; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP16:%[0-9]+]]>
+; IC2-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi>.1 = phi (find-last) ir<-1>, vp<[[VP17:%[0-9]+]]>
; IC2-TF-NEXT: WIDEN-PHI vp<[[VP4:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP14:%[0-9]+]]>, vector.body ]
; IC2-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP15:%[0-9]+]]>, vector.body ]
; IC2-TF-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<%index>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
index d20e9ae47cd45..a0b73d45acf10 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
@@ -273,7 +273,7 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi ir<0>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) ir<0>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
index cd95f5cb99507..137ea602e99b5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions.ll
@@ -22,7 +22,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi fast vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -95,7 +95,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi fast vp<[[VP3]]>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP3]]>, ir<%red.next>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -170,7 +170,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi nnan ninf nsz vp<[[VP3]]>, ir<%muladd>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi (fmuladd) nnan ninf nsz vp<[[VP3]]>, ir<%muladd>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -187,7 +187,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) nnan ninf nsz ir<%muladd>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (fmuladd, in-loop) nnan ninf nsz ir<%muladd>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -252,7 +252,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi ir<-9223372036854775808>, ir<%cond>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, ir<%cond>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>
@@ -330,7 +330,7 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -406,7 +406,7 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -489,7 +489,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -578,7 +578,7 @@ define i64 @print_extended_sub_reduction(ptr nocapture readonly %x, ptr nocaptur
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -654,7 +654,7 @@ define i32 @print_mulacc_sub(ptr %a, ptr %b) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (sub) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.a>
@@ -740,7 +740,7 @@ define i32 @print_mulacc_negated(ptr %a, ptr %b) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%accum> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.a>
@@ -828,7 +828,7 @@ define i64 @print_mulacc_sub_extended(ptr nocapture readonly %x, ptr nocapture r
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -917,7 +917,7 @@ define i64 @print_mulacc_duplicate_extends(ptr nocapture readonly %x, ptr nocapt
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
@@ -999,7 +999,7 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
@@ -1076,7 +1076,7 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
@@ -1155,7 +1155,7 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
@@ -1234,7 +1234,7 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP9:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
@@ -1314,7 +1314,7 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi vp<[[VP5]]>, vp<[[VP10:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP10:%[0-9]+]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer vp<%next.gep>
@@ -1400,7 +1400,7 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi vp<[[VP3]]>, vp<[[VP5:%[0-9]+]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP3]]>, vp<[[VP5:%[0-9]+]]>
; CHECK-NEXT: CLONE ir<%load> = load ir<%a>
; CHECK-NEXT: EXPRESSION vp<[[VP5]]> = ir<%res2> + reduce.add (ir<%mul> zext to i64)
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
@@ -1486,7 +1486,7 @@ define i32 @print_umax_reduction(ptr %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (umax) ir<0>, ir<%red.next>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%y>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%gep>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
index ba62ebcd29ffb..c4fbd1ecf44b2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
@@ -33,7 +33,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; CHECK: Cost of 1 for VF 2: exit condition instruction %exitcond = icmp eq i32 %lftr.wideiv, %n
; CHECK: Cost of 0 for VF 2: exit condition instruction %lftr.wideiv = trunc i64 %indvars.iv.next to i32
; CHECK: Cost of 0 for VF 2: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%sum.013> = phi vp<{{.+}}>, vp<[[EXT:%.+]]>
+; CHECK: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%sum.013> = phi (add) vp<{{.+}}>, vp<[[EXT:%.+]]>
; CHECK: Cost of 0 for VF 2: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
; CHECK: Cost of 0 for VF 2: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]>
; CHECK: Cost of 0 for VF 2: vp<[[VECP1:%.+]]> = vector-pointer inbounds ir<%arrayidx>
>From d515899e391a4bfefb050a852c7cb956ba0eab7c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 2 Apr 2026 23:47:27 +0800
Subject: [PATCH 2/2] Keep definition in VPlanRecipes
---
llvm/include/llvm/Analysis/IVDescriptors.h | 84 ------------------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 87 ++++++++++++++++++-
2 files changed, 85 insertions(+), 86 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 4948effe6fc78..8383da0f9e879 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -458,90 +458,6 @@ class InductionDescriptor {
SmallVector<Instruction *, 2> RedundantCasts;
};
-inline raw_ostream &operator<<(raw_ostream &OS, const RecurKind &Kind) {
- switch (Kind) {
- case RecurKind::None:
- OS << "none";
- break;
- case RecurKind::Add:
- OS << "add";
- break;
- case RecurKind::Sub:
- OS << "sub";
- break;
- case RecurKind::AddChainWithSubs:
- OS << "add-chain-with-subs";
- break;
- case RecurKind::Mul:
- OS << "mul";
- break;
- case RecurKind::Or:
- OS << "or";
- break;
- case RecurKind::And:
- OS << "and";
- break;
- case RecurKind::Xor:
- OS << "xor";
- break;
- case RecurKind::SMin:
- OS << "smin";
- break;
- case RecurKind::SMax:
- OS << "smax";
- break;
- case RecurKind::UMin:
- OS << "umin";
- break;
- case RecurKind::UMax:
- OS << "umax";
- break;
- case RecurKind::FAdd:
- OS << "fadd";
- break;
- case RecurKind::FMul:
- OS << "fmul";
- break;
- case RecurKind::FMin:
- OS << "fmin";
- break;
- case RecurKind::FMax:
- OS << "fmax";
- break;
- case RecurKind::FMinNum:
- OS << "fminnum";
- break;
- case RecurKind::FMaxNum:
- OS << "fmaxnum";
- break;
- case RecurKind::FMinimum:
- OS << "fminimum";
- break;
- case RecurKind::FMaximum:
- OS << "fmaximum";
- break;
- case RecurKind::FMinimumNum:
- OS << "fminimumnum";
- break;
- case RecurKind::FMaximumNum:
- OS << "fmaximumnum";
- break;
- case RecurKind::FMulAdd:
- OS << "fmuladd";
- break;
- case RecurKind::AnyOf:
- OS << "any-of";
- break;
- case RecurKind::FindIV:
- OS << "find-iv";
- break;
- case RecurKind::FindLast:
- OS << "find-last";
- break;
- }
- return OS;
-}
-
} // end namespace llvm
#endif // LLVM_ANALYSIS_IVDESCRIPTORS_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4ce6af4038889..d7c3e6bf5e165 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2200,6 +2200,89 @@ bool VPIRFlags::hasRequiredFlagsForOpcode(unsigned Opcode) const {
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+static void printRecurKind(raw_ostream &OS, const RecurKind &Kind) {
+ switch (Kind) {
+ case RecurKind::None:
+ OS << "none";
+ break;
+ case RecurKind::Add:
+ OS << "add";
+ break;
+ case RecurKind::Sub:
+ OS << "sub";
+ break;
+ case RecurKind::AddChainWithSubs:
+ OS << "add-chain-with-subs";
+ break;
+ case RecurKind::Mul:
+ OS << "mul";
+ break;
+ case RecurKind::Or:
+ OS << "or";
+ break;
+ case RecurKind::And:
+ OS << "and";
+ break;
+ case RecurKind::Xor:
+ OS << "xor";
+ break;
+ case RecurKind::SMin:
+ OS << "smin";
+ break;
+ case RecurKind::SMax:
+ OS << "smax";
+ break;
+ case RecurKind::UMin:
+ OS << "umin";
+ break;
+ case RecurKind::UMax:
+ OS << "umax";
+ break;
+ case RecurKind::FAdd:
+ OS << "fadd";
+ break;
+ case RecurKind::FMul:
+ OS << "fmul";
+ break;
+ case RecurKind::FMin:
+ OS << "fmin";
+ break;
+ case RecurKind::FMax:
+ OS << "fmax";
+ break;
+ case RecurKind::FMinNum:
+ OS << "fminnum";
+ break;
+ case RecurKind::FMaxNum:
+ OS << "fmaxnum";
+ break;
+ case RecurKind::FMinimum:
+ OS << "fminimum";
+ break;
+ case RecurKind::FMaximum:
+ OS << "fmaximum";
+ break;
+ case RecurKind::FMinimumNum:
+ OS << "fminimumnum";
+ break;
+ case RecurKind::FMaximumNum:
+ OS << "fmaximumnum";
+ break;
+ case RecurKind::FMulAdd:
+ OS << "fmuladd";
+ break;
+ case RecurKind::AnyOf:
+ OS << "any-of";
+ break;
+ case RecurKind::FindIV:
+ OS << "find-iv";
+ break;
+ case RecurKind::FindLast:
+ OS << "find-last";
+ break;
+ }
+}
+
void VPIRFlags::printFlags(raw_ostream &O) const {
switch (OpType) {
case OperationType::Cmp:
@@ -2248,7 +2331,7 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
break;
case OperationType::ReductionOp: {
O << " (";
- O << getRecurKind();
+ printRecurKind(O, getRecurKind());
if (isReductionInLoop())
O << ", in-loop";
if (isReductionOrdered())
@@ -4630,7 +4713,7 @@ void VPReductionPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
printAsOperand(O, SlotTracker);
O << " = phi (";
- O << Kind;
+ printRecurKind(O, Kind);
O << ")";
printFlags(O);
printOperands(O, SlotTracker);
More information about the llvm-commits
mailing list