[llvm] [LV] Reduce register pressure of RISC-V outer loop reduction. (PR #208621)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 17:51:33 PDT 2026
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/208621
>From f28f0bf916c2cc05be24e7c33038fe148af59467 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 6 Jul 2026 23:25:43 -0700
Subject: [PATCH 1/4] Precommit tests.
---
.../LoopVectorize/RISCV/reg-usage.ll | 65 +++++++++++++++++++
1 file changed, 65 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 31ccd43911139..86652c8f02a75 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -126,3 +126,68 @@ for.body:
%exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
}
+
+
+define void @red(ptr %base.0, ptr %base.1, ptr %base.2, ptr %base.3, i64 %end) {
+; CHECK-LABEL: 'red'
+; CHECK: LV(REG): Calculating max register usage:
+; CHECK: LV(REG): VF = 1
+; CHECK: LV(REG): Found max usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 10 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): Calculating max register usage:
+; CHECK: LV(REG): VF = vscale x 1
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): VF = vscale x 2
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 16 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): VF = vscale x 4
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 32 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): VF = vscale x 8
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 64 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): Cost of 64 from 32 spills of RISCV::VRRC
+;
+entry:
+ br label %loop.body
+
+loop.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.body ]
+ %red.0 = phi i64 [ 0, %entry ], [ %red.0.next, %loop.body ]
+ %red.1 = phi i64 [ 0, %entry ], [ %red.1.next, %loop.body ]
+ %red.2 = phi i64 [ 0, %entry ], [ %red.2.next, %loop.body ]
+ %red.3 = phi i64 [ 0, %entry ], [ %red.3.next, %loop.body ]
+ %ptr.0 = getelementptr i8, ptr %base.0, i64 %iv
+ %0 = load i64, ptr %ptr.0, align 8
+ %ptr.1 = getelementptr i8, ptr %base.1, i64 %iv
+ %1 = load i64, ptr %ptr.1, align 8
+ %red.0.next = tail call i64 @llvm.smin.i64(i64 %0, i64 %red.0)
+ %red.2.next = tail call i64 @llvm.smax.i64(i64 %0, i64 %red.2)
+ %red.1.next = tail call i64 @llvm.smin.i64(i64 %1, i64 %red.1)
+ %red.3.next = tail call i64 @llvm.smax.i64(i64 %1, i64 %red.3)
+ %iv.next = add i64 %iv, 1
+ %exitcond = icmp eq i64 %iv, %end
+ br i1 %exitcond, label %loop.exit, label %loop.body
+
+loop.exit:
+ store i64 %red.0.next, ptr %base.0
+ store i64 %red.1.next, ptr %base.1
+ store i64 %red.2.next, ptr %base.2
+ store i64 %red.3.next, ptr %base.3
+ ret void
+}
>From ac433b02189f60b4f20ede19e08c3e27210dd773 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Sun, 26 Jul 2026 23:50:02 -0700
Subject: [PATCH 2/4] [LV] Wrap inloop operation and predication to reduce reg
pressure.
This patch warp the in-loop operation and the predication (vp.merge)
into one ExpressionRecipe to reduce the register pressure since the
vp.merge can be optmized out.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 6 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 56 ++++++++++++++++++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 42 ++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 4 ++
.../RISCV/reg-usage-maxbandwidth.ll | 47 +++++++++++++---
.../LoopVectorize/RISCV/reg-usage.ll | 18 +++---
.../RISCV/tail-folding-reduction-cost.ll | 12 ++--
.../RISCV/vplan-vp-intrinsics-reduction.ll | 3 +-
.../VPlan/vplan-print-before-after-all.ll | 1 +
10 files changed, 163 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 45edf702d7092..f4c2287b2d204 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6549,6 +6549,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
+ RUN_VPLAN_PASS(VPlanTransforms::prepareForCostModel, *Plan);
RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
VPlans.push_back(std::move(Plan));
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index aa93dbaa65170..1b67e3cea78f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3572,6 +3572,9 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
/// extended vector operands, negating the multiplication, performing a
/// reduction.add on the result, and adding the scalar result to a chain.
ExtNegatedMulAccReduction,
+ /// Represent an inloop operations with tail-floded select for the outer
+ /// loop reduction.
+ TailFoldedInLoopOp,
};
/// Type of the expression.
@@ -3629,6 +3632,9 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
} else
assert(Neg->getOpcode() == Instruction::FNeg && "Unexpected opcode");
}
+ VPExpressionRecipe(VPSingleDefRecipe *InLoopOp, VPSingleDefRecipe *VPMerge)
+ : VPExpressionRecipe(ExpressionTypes::TailFoldedInLoopOp,
+ {InLoopOp, VPMerge}) {}
~VPExpressionRecipe() override {
SmallPtrSet<VPSingleDefRecipe *, 4> ExpressionRecipesSeen;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ae9001dc8c67f..a92dded0a6750 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3648,9 +3648,11 @@ VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
: VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {},
- cast<VPReductionRecipe>(ExpressionRecipes.back())
- ->getChainOp()
- ->getScalarType()),
+ ExpressionType == ExpressionTypes::TailFoldedInLoopOp
+ ? ExpressionRecipes.back()->getScalarType()
+ : cast<VPReductionRecipe>(ExpressionRecipes.back())
+ ->getChainOp()
+ ->getScalarType()),
ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
assert(!ExpressionRecipes.empty() && "Nothing to combine?");
assert(
@@ -3723,6 +3725,14 @@ SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
+ // Handle expression recipe without in-loop reduction.
+ if (ExpressionType == ExpressionTypes::TailFoldedInLoopOp) {
+ InstructionCost Cost = 0;
+ for (auto *R : ExpressionRecipes)
+ Cost += R->cost(VF, Ctx);
+ return Cost;
+ }
+
Type *RedTy = this->getScalarType();
auto *SrcVecTy =
cast<VectorType>(toVectorTy(getOperand(0)->getScalarType(), VF));
@@ -3794,6 +3804,8 @@ InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
Instruction::ZExt,
Opcode, RedTy, SrcVecTy, Ctx.CostKind);
}
+ case ExpressionTypes::TailFoldedInLoopOp:
+ llvm_unreachable("TailFoldedInLoopOp should be handled early");
}
llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
}
@@ -3824,6 +3836,42 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "EXPRESSION ";
printAsOperand(O, SlotTracker);
O << " = ";
+ // Handle the tail-folded in-loop operation
+ if (ExpressionType == ExpressionTypes::TailFoldedInLoopOp) {
+ VPSingleDefRecipe *InLoopOp = ExpressionRecipes[0];
+ unsigned NumInLoopOps = InLoopOp->getNumOperands();
+ O << "vp.merge ";
+ getOperand(NumInLoopOps)->printAsOperand(O, SlotTracker);
+ O << ", ";
+ auto PrintInLoopOperands = [&]() {
+ O << "(";
+ for (unsigned Idx = 0; Idx != NumInLoopOps; ++Idx) {
+ if (Idx != 0)
+ O << ", ";
+ getOperand(Idx)->printAsOperand(O, SlotTracker);
+ }
+ O << ")";
+ };
+
+ if (auto *WidenIntrinsic = dyn_cast<VPWidenIntrinsicRecipe>(InLoopOp)) {
+ O << WidenIntrinsic->getIntrinsicName();
+ WidenIntrinsic->printFlags(O);
+ PrintInLoopOperands();
+ } else if (auto *Widen = dyn_cast<VPWidenRecipe>(InLoopOp)) {
+ O << Instruction::getOpcodeName(Widen->getOpcode());
+ Widen->printFlags(O);
+ PrintInLoopOperands();
+ } else {
+ llvm_unreachable("Unsupported in-loop recipe for tail-folded expression");
+ }
+ O << ", ";
+ getOperand(NumInLoopOps + 1)->printAsOperand(O, SlotTracker);
+ O << ", ";
+ getOperand(NumInLoopOps + 2)->printAsOperand(O, SlotTracker);
+
+ return;
+ }
+
auto *Red = cast<VPReductionRecipe>(ExpressionRecipes.back());
unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
VPValue *Mask = getOperand(getNumOperands() - 1);
@@ -3918,6 +3966,8 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << ")";
break;
}
+ default:
+ llvm_unreachable("Unhandled VPExpressionRecipe::ExpressionTypes enum");
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dbdac9a27470d..1063418dfc11b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6123,3 +6123,45 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
}
}
}
+
+static void tryToCreateAbstractInLoopOp(VPWidenIntrinsicRecipe *VPMerge) {
+ // Only convert the in-loop operations with tail-folding predication to
+ // expression recipe.
+ if (any_of(VPMerge->users(), [](VPUser *U) {
+ if (auto *R = dyn_cast<VPSingleDefRecipe>(U))
+ return !isa<VPReductionPHIRecipe>(R) &&
+ !match(R, m_ComputeReductionResult(m_VPValue()));
+ return true;
+ }))
+ return;
+
+ VPSingleDefRecipe *InLoopOp =
+ dyn_cast<VPSingleDefRecipe>(VPMerge->getOperand(1));
+ // TODO: Support cmp-select reductions.
+ if (!InLoopOp || !isa<VPWidenRecipe, VPWidenIntrinsicRecipe>(InLoopOp) ||
+ match(InLoopOp, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+ return;
+
+ VPBasicBlock *VPBB = VPMerge->getParent();
+ auto IP = std::next(VPMerge->getIterator());
+ auto *Expr = new VPExpressionRecipe(InLoopOp, VPMerge);
+ Expr->insertBefore(*VPBB, IP);
+ VPMerge->replaceAllUsesWith(Expr);
+}
+
+void VPlanTransforms::prepareForCostModel(VPlan &Plan) {
+ if (Plan.hasScalarVFOnly())
+ return;
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ // Warp in-loop operations and the vp.merge (cleanup tail poison) to
+ // expression recipes since the vp.merge will be optmized out in the
+ // backend.
+ if (match(&R, m_Intrinsic<Intrinsic::vp_merge>(m_VPValue(), m_VPValue(),
+ m_VPValue(), m_VPValue())))
+ tryToCreateAbstractInLoopOp(cast<VPWidenIntrinsicRecipe>(&R));
+ }
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 0469324023f52..dea8280a59774 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -649,6 +649,10 @@ struct VPlanTransforms {
static void narrowInductionTruncates(VPlan &Plan, VFRange &Range,
const TargetTransformInfo &TTI,
PredicatedScalarEvolution &PSE);
+
+ /// Wrap recipes in \p Plan into expression recipes that can help
+ /// cost/register pressure estimation.
+ static void prepareForCostModel(VPlan &Plan);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
index 41f1a9b45b7ff..e02f8e452f387 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
@@ -3,13 +3,46 @@
; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-maximize-bandwidth -debug-only=loop-vectorize,vplan -disable-output -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
define i32 @dotp(ptr %a, ptr %b) {
-; CHECK-REGS-VP: LV(REG): VF = vscale x 16
-; CHECK-REGS-VP-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
-; CHECK-REGS-VP-NEXT: LV(REG): Found invariant usage: 1 item
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK-REGS-VP: LV: Selecting VF: vscale x 16.
+; CHECK-REGS-VP-LABEL: 'dotp'
+; CHECK-REGS-VP: LV(REG): Calculating max register usage:
+; CHECK-REGS-VP: LV(REG): VF = 1
+; CHECK-REGS-VP: LV(REG): Found max usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): Calculating max register usage:
+; CHECK-REGS-VP: LV(REG): VF = vscale x 1
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 3 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 2
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 3 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 4
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 8
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 16
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV: Selecting VF: vscale x 16.
+;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 86652c8f02a75..4259fdd4f35d6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -139,29 +139,29 @@ define void @red(ptr %base.0, ptr %base.1, ptr %base.2, ptr %base.3, i64 %end) {
; CHECK: LV(REG): Calculating max register usage:
; CHECK: LV(REG): VF = vscale x 1
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 7 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK: LV(REG): VF = vscale x 2
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 16 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 13 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK: LV(REG): VF = vscale x 4
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 32 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 26 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK: LV(REG): VF = vscale x 8
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 64 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 52 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK: LV(REG): Cost of 64 from 32 spills of RISCV::VRRC
+; CHECK: LV(REG): Cost of 40 from 20 spills of RISCV::VRRC
;
entry:
br label %loop.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
index 1c644aadfd8ad..3615797170da1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*(WIDEN-REDUCTION-PHI|WIDEN-INTRINSIC)" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*(WIDEN-REDUCTION-PHI|EXPRESSION)" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize --disable-output \
; RUN: -tail-folding-policy=prefer-fold-tail -vectorizer-maximize-bandwidth \
@@ -8,15 +8,15 @@
define i64 @add(ptr %a, i64 %n, i64 %start) {
; CHECK-LABEL: 'add'
; CHECK: Cost of 0 for VF vscale x 1: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2:%[0-9]+]]>, vp<[[VP9:%[0-9]+]]>
-; CHECK: Cost of 0 for VF vscale x 1: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 1 for VF vscale x 1: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 2: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 0 for VF vscale x 2: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 2 for VF vscale x 2: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 4: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 0 for VF vscale x 4: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 4 for VF vscale x 4: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 8: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 0 for VF vscale x 8: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 8 for VF vscale x 8: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 16: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 16 for VF vscale x 16: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 32 for VF vscale x 16: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
index 4b84e9bcb687f..590fecddfd2ab 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -91,8 +91,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP7]]>
; IF-EVL-OUTLOOP-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx>, ir<1>
; IF-EVL-OUTLOOP-NEXT: WIDEN ir<%0> = vp.load vp<[[VP8]]>, vp<%evl>
-; IF-EVL-OUTLOOP-NEXT: WIDEN ir<%add> = add ir<%0>, ir<%rdx>
-; IF-EVL-OUTLOOP-NEXT: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; IF-EVL-OUTLOOP-NEXT: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%0>, ir<%rdx>), ir<%rdx>, vp<%evl>
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
; IF-EVL-OUTLOOP-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP5]]>
; IF-EVL-OUTLOOP-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 5fd844e186e44..2089e33eacfea 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -68,6 +68,7 @@
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::narrowInterleaveGroups
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeHeaderMask
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::prepareForCostModel
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printOptimizedVPlan
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMinimumIterationCheck
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceWideCanonicalIVWithWideIV
>From 4d425128d7f7e9c6f3ea864ebbe923e0e0e7e52b Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Wed, 9 Sep 2026 23:48:28 -0700
Subject: [PATCH 3/4] Address comments.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 10 ++---
.../Vectorize/VPlanEVLTailFolding.cpp | 43 ++++++++++++++++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 45 ++++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 42 -----------------
.../Transforms/Vectorize/VPlanTransforms.h | 2 +-
.../RISCV/tail-folding-reduction-cost.ll | 2 +-
.../VPlan/vplan-print-before-after-all.ll | 1 -
8 files changed, 74 insertions(+), 73 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f4c2287b2d204..390ca908f6ad4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6538,6 +6538,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
+ RUN_VPLAN_PASS(VPlanTransforms::foldPredicateMerge, *Plan);
}
if (auto P =
@@ -6549,7 +6550,6 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
- RUN_VPLAN_PASS(VPlanTransforms::prepareForCostModel, *Plan);
RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
VPlans.push_back(std::move(Plan));
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1b67e3cea78f7..a3ab861c1c44f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3572,9 +3572,10 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
/// extended vector operands, negating the multiplication, performing a
/// reduction.add on the result, and adding the scalar result to a chain.
ExtNegatedMulAccReduction,
- /// Represent an inloop operations with tail-floded select for the outer
- /// loop reduction.
- TailFoldedInLoopOp,
+ /// Represent a chain of recipes that the last recipe will be optimized
+ /// away.
+ /// For example A(B(C(...))), LV won't calculate the cost for recipe A.
+ FoldedOp,
};
/// Type of the expression.
@@ -3633,8 +3634,7 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
assert(Neg->getOpcode() == Instruction::FNeg && "Unexpected opcode");
}
VPExpressionRecipe(VPSingleDefRecipe *InLoopOp, VPSingleDefRecipe *VPMerge)
- : VPExpressionRecipe(ExpressionTypes::TailFoldedInLoopOp,
- {InLoopOp, VPMerge}) {}
+ : VPExpressionRecipe(ExpressionTypes::FoldedOp, {InLoopOp, VPMerge}) {}
~VPExpressionRecipe() override {
SmallPtrSet<VPSingleDefRecipe *, 4> ExpressionRecipesSeen;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index b6ede1c1edc7f..e8dd79ce5127d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -653,3 +653,46 @@ void VPlanTransforms::convertEVLExitCond(VPlan &Plan) {
LatchBr->setOperand(
0, Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy)));
}
+
+static void wrapPredicateMerge(VPWidenIntrinsicRecipe *VPMerge) {
+ assert(VPMerge->getVectorIntrinsicID() == Intrinsic::vp_merge);
+ VPSingleDefRecipe *InLoopOp =
+ dyn_cast<VPSingleDefRecipe>(VPMerge->getOperand(1));
+ // TODO: Support cmp-select reductions.
+ if (!InLoopOp || !isa<VPWidenRecipe, VPWidenIntrinsicRecipe>(InLoopOp) ||
+ match(InLoopOp, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+ return;
+
+ // Only convert the in-loop operations with tail-folding predication to
+ // expression recipe.
+ if (!all_of(VPMerge->users(), [](VPUser *U) {
+ if (auto *R = dyn_cast<VPSingleDefRecipe>(U))
+ return isa<VPReductionPHIRecipe>(R) ||
+ match(R, m_ComputeReductionResult(m_VPValue()));
+ return false;
+ }))
+ return;
+
+ VPBasicBlock *VPBB = VPMerge->getParent();
+ auto IP = std::next(VPMerge->getIterator());
+ auto *Expr = new VPExpressionRecipe(InLoopOp, VPMerge);
+ Expr->insertBefore(*VPBB, IP);
+ VPMerge->replaceAllUsesWith(Expr);
+}
+
+void VPlanTransforms::foldPredicateMerge(VPlan &Plan) {
+ if (Plan.hasScalarVFOnly())
+ return;
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ // Wrap in-loop operations and the vp.merge (cleanup tail poison) to
+ // expression recipes since the vp.merge will be optmized out in the
+ // backend.
+ if (match(&R, m_Intrinsic<Intrinsic::vp_merge>(m_VPValue(), m_VPValue(),
+ m_VPValue(), m_VPValue())))
+ wrapPredicateMerge(cast<VPWidenIntrinsicRecipe>(&R));
+ }
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index a92dded0a6750..2ef52cd484ea0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3648,11 +3648,14 @@ VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
: VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {},
- ExpressionType == ExpressionTypes::TailFoldedInLoopOp
- ? ExpressionRecipes.back()->getScalarType()
- : cast<VPReductionRecipe>(ExpressionRecipes.back())
- ->getChainOp()
- ->getScalarType()),
+ [&]() {
+ if (ExpressionType == ExpressionTypes::FoldedOp)
+ return ExpressionRecipes.back()->getScalarType();
+ return cast<VPReductionRecipe>(
+ ExpressionRecipes.back())
+ ->getChainOp()
+ ->getScalarType();
+ }()),
ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
assert(!ExpressionRecipes.empty() && "Nothing to combine?");
assert(
@@ -3725,10 +3728,10 @@ SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
- // Handle expression recipe without in-loop reduction.
- if (ExpressionType == ExpressionTypes::TailFoldedInLoopOp) {
+ // The last recipes in the chain will be optimized away, so igonre the cost.
+ if (ExpressionType == ExpressionTypes::FoldedOp) {
InstructionCost Cost = 0;
- for (auto *R : ExpressionRecipes)
+ for (auto *R : drop_end(ExpressionRecipes))
Cost += R->cost(VF, Ctx);
return Cost;
}
@@ -3804,8 +3807,8 @@ InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
Instruction::ZExt,
Opcode, RedTy, SrcVecTy, Ctx.CostKind);
}
- case ExpressionTypes::TailFoldedInLoopOp:
- llvm_unreachable("TailFoldedInLoopOp should be handled early");
+ default:
+ llvm_unreachable("Unsupported VPExpressionRecipe::ExpressionTypes enum");
}
llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
}
@@ -3836,8 +3839,7 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "EXPRESSION ";
printAsOperand(O, SlotTracker);
O << " = ";
- // Handle the tail-folded in-loop operation
- if (ExpressionType == ExpressionTypes::TailFoldedInLoopOp) {
+ if (ExpressionType == ExpressionTypes::FoldedOp) {
VPSingleDefRecipe *InLoopOp = ExpressionRecipes[0];
unsigned NumInLoopOps = InLoopOp->getNumOperands();
O << "vp.merge ";
@@ -3845,11 +3847,10 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << ", ";
auto PrintInLoopOperands = [&]() {
O << "(";
- for (unsigned Idx = 0; Idx != NumInLoopOps; ++Idx) {
- if (Idx != 0)
- O << ", ";
- getOperand(Idx)->printAsOperand(O, SlotTracker);
- }
+ interleaveComma(make_range(op_begin(), op_begin() + NumInLoopOps), O,
+ [&O, &SlotTracker](VPValue *Op) {
+ Op->printAsOperand(O, SlotTracker);
+ });
O << ")";
};
@@ -3861,13 +3862,13 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Instruction::getOpcodeName(Widen->getOpcode());
Widen->printFlags(O);
PrintInLoopOperands();
- } else {
- llvm_unreachable("Unsupported in-loop recipe for tail-folded expression");
}
O << ", ";
- getOperand(NumInLoopOps + 1)->printAsOperand(O, SlotTracker);
- O << ", ";
- getOperand(NumInLoopOps + 2)->printAsOperand(O, SlotTracker);
+ interleaveComma(make_range(op_begin() + NumInLoopOps + 1,
+ op_begin() + NumInLoopOps + 3),
+ O, [&O, &SlotTracker](VPValue *Op) {
+ Op->printAsOperand(O, SlotTracker);
+ });
return;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1063418dfc11b..dbdac9a27470d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6123,45 +6123,3 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
}
}
}
-
-static void tryToCreateAbstractInLoopOp(VPWidenIntrinsicRecipe *VPMerge) {
- // Only convert the in-loop operations with tail-folding predication to
- // expression recipe.
- if (any_of(VPMerge->users(), [](VPUser *U) {
- if (auto *R = dyn_cast<VPSingleDefRecipe>(U))
- return !isa<VPReductionPHIRecipe>(R) &&
- !match(R, m_ComputeReductionResult(m_VPValue()));
- return true;
- }))
- return;
-
- VPSingleDefRecipe *InLoopOp =
- dyn_cast<VPSingleDefRecipe>(VPMerge->getOperand(1));
- // TODO: Support cmp-select reductions.
- if (!InLoopOp || !isa<VPWidenRecipe, VPWidenIntrinsicRecipe>(InLoopOp) ||
- match(InLoopOp, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
- return;
-
- VPBasicBlock *VPBB = VPMerge->getParent();
- auto IP = std::next(VPMerge->getIterator());
- auto *Expr = new VPExpressionRecipe(InLoopOp, VPMerge);
- Expr->insertBefore(*VPBB, IP);
- VPMerge->replaceAllUsesWith(Expr);
-}
-
-void VPlanTransforms::prepareForCostModel(VPlan &Plan) {
- if (Plan.hasScalarVFOnly())
- return;
-
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- // Warp in-loop operations and the vp.merge (cleanup tail poison) to
- // expression recipes since the vp.merge will be optmized out in the
- // backend.
- if (match(&R, m_Intrinsic<Intrinsic::vp_merge>(m_VPValue(), m_VPValue(),
- m_VPValue(), m_VPValue())))
- tryToCreateAbstractInLoopOp(cast<VPWidenIntrinsicRecipe>(&R));
- }
- }
-}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index dea8280a59774..1c7d512a9656e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -652,7 +652,7 @@ struct VPlanTransforms {
/// Wrap recipes in \p Plan into expression recipes that can help
/// cost/register pressure estimation.
- static void prepareForCostModel(VPlan &Plan);
+ static void foldPredicateMerge(VPlan &Plan);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
index 3615797170da1..ef959f6f582f1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
@@ -16,7 +16,7 @@ define i64 @add(ptr %a, i64 %n, i64 %start) {
; CHECK: Cost of 0 for VF vscale x 8: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
; CHECK: Cost of 8 for VF vscale x 8: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 16: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 32 for VF vscale x 16: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
+; CHECK: Cost of 16 for VF vscale x 16: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 2089e33eacfea..5fd844e186e44 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -68,7 +68,6 @@
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::narrowInterleaveGroups
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeHeaderMask
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::prepareForCostModel
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printOptimizedVPlan
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMinimumIterationCheck
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceWideCanonicalIVWithWideIV
>From c1c98a097dbbccb481932d8e4f3c84d503f6e9d0 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Tue, 29 Sep 2026 22:06:46 -0700
Subject: [PATCH 4/4] Address comments.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
.../Vectorize/VPlanEVLTailFolding.cpp | 66 ++++++++++---------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 13 +++-
.../Transforms/Vectorize/VPlanTransforms.h | 6 +-
.../RISCV/tail-folding-complex-mask.ll | 4 +-
5 files changed, 53 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 390ca908f6ad4..b9d2d16d5dd7f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6538,7 +6538,8 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
Config.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
- RUN_VPLAN_PASS(VPlanTransforms::foldPredicateMerge, *Plan);
+ VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
+ RUN_VPLAN_PASS(VPlanTransforms::foldPredicateMerge, *Plan, CostCtx);
}
if (auto P =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
index e8dd79ce5127d..2c5c173395395 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTailFolding.cpp
@@ -654,33 +654,7 @@ void VPlanTransforms::convertEVLExitCond(VPlan &Plan) {
0, Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy)));
}
-static void wrapPredicateMerge(VPWidenIntrinsicRecipe *VPMerge) {
- assert(VPMerge->getVectorIntrinsicID() == Intrinsic::vp_merge);
- VPSingleDefRecipe *InLoopOp =
- dyn_cast<VPSingleDefRecipe>(VPMerge->getOperand(1));
- // TODO: Support cmp-select reductions.
- if (!InLoopOp || !isa<VPWidenRecipe, VPWidenIntrinsicRecipe>(InLoopOp) ||
- match(InLoopOp, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
- return;
-
- // Only convert the in-loop operations with tail-folding predication to
- // expression recipe.
- if (!all_of(VPMerge->users(), [](VPUser *U) {
- if (auto *R = dyn_cast<VPSingleDefRecipe>(U))
- return isa<VPReductionPHIRecipe>(R) ||
- match(R, m_ComputeReductionResult(m_VPValue()));
- return false;
- }))
- return;
-
- VPBasicBlock *VPBB = VPMerge->getParent();
- auto IP = std::next(VPMerge->getIterator());
- auto *Expr = new VPExpressionRecipe(InLoopOp, VPMerge);
- Expr->insertBefore(*VPBB, IP);
- VPMerge->replaceAllUsesWith(Expr);
-}
-
-void VPlanTransforms::foldPredicateMerge(VPlan &Plan) {
+void VPlanTransforms::foldPredicateMerge(VPlan &Plan, VPCostContext &CostCtx) {
if (Plan.hasScalarVFOnly())
return;
@@ -690,9 +664,41 @@ void VPlanTransforms::foldPredicateMerge(VPlan &Plan) {
// Wrap in-loop operations and the vp.merge (cleanup tail poison) to
// expression recipes since the vp.merge will be optmized out in the
// backend.
- if (match(&R, m_Intrinsic<Intrinsic::vp_merge>(m_VPValue(), m_VPValue(),
- m_VPValue(), m_VPValue())))
- wrapPredicateMerge(cast<VPWidenIntrinsicRecipe>(&R));
+ VPValue *True;
+ if (!match(&R,
+ m_Intrinsic<Intrinsic::vp_merge>(m_VPValue(), m_VPValue(True),
+ m_VPValue(), m_VPValue())))
+ continue;
+ auto *VPMerge = cast<VPWidenIntrinsicRecipe>(&R);
+ if (!True->hasOneUse())
+ continue;
+
+ // Query TTI to check if vp.merge can be folded.
+ if (VPMerge->computeCost(*Plan.vectorFactors().begin(), CostCtx) != 0)
+ continue;
+
+ VPSingleDefRecipe *InLoopOp =
+ dyn_cast<VPSingleDefRecipe>(VPMerge->getOperand(1));
+ // TODO: Support cmp-select reductions.
+ if (!InLoopOp || !isa<VPWidenRecipe, VPWidenIntrinsicRecipe>(InLoopOp) ||
+ match(InLoopOp, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+ continue;
+
+ // Only convert the in-loop operations with tail-folding predication to
+ // expression recipe.
+ if (!all_of(VPMerge->users(), [](VPUser *U) {
+ if (auto *R = dyn_cast<VPSingleDefRecipe>(U))
+ return isa<VPReductionPHIRecipe>(R) ||
+ match(R, m_ComputeReductionResult(m_VPValue()));
+ return false;
+ }))
+ continue;
+
+ VPBasicBlock *VPBB = VPMerge->getParent();
+ auto IP = std::next(VPMerge->getIterator());
+ auto *Expr = new VPExpressionRecipe(InLoopOp, VPMerge);
+ Expr->insertBefore(*VPBB, IP);
+ VPMerge->replaceAllUsesWith(Expr);
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2ef52cd484ea0..d95e5dbdb182f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2409,12 +2409,19 @@ InstructionCost VPWidenIntrinsicRecipe::computeCallCost(
});
VectorInstrContext VIC = VectorInstrContext::None;
- for (const VPValue *Op : Operands)
+ for (const VPValue *Op : Operands) {
if (isa<VPWidenRecipe>(Op) &&
Instruction::isBinaryOp(cast<VPWidenRecipe>(Op)->getOpcode())) {
VIC = VectorInstrContext::BinaryOp;
break;
}
+ if (isa<VPWidenIntrinsicRecipe>(Op) &&
+ isa_and_nonnull<MinMaxIntrinsic>(
+ cast<VPWidenIntrinsicRecipe>(Op)->getUnderlyingValue())) {
+ VIC = VectorInstrContext::BinaryOp;
+ break;
+ }
+ }
// TODO: Rework TTI interface to avoid reliance on underlying IntrinsicInst.
IntrinsicCostAttributes CostAttrs(
@@ -3807,8 +3814,8 @@ InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
Instruction::ZExt,
Opcode, RedTy, SrcVecTy, Ctx.CostKind);
}
- default:
- llvm_unreachable("Unsupported VPExpressionRecipe::ExpressionTypes enum");
+ case ExpressionTypes::FoldedOp:
+ llvm_unreachable("Folded expression should be handled eariler");
}
llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 1c7d512a9656e..ec9e73089afa4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -650,9 +650,9 @@ struct VPlanTransforms {
const TargetTransformInfo &TTI,
PredicatedScalarEvolution &PSE);
- /// Wrap recipes in \p Plan into expression recipes that can help
- /// cost/register pressure estimation.
- static void foldPredicateMerge(VPlan &Plan);
+ /// Wrap recipes in \p Plan into expression recipes based on \p CostCtx to
+ /// reduce cost/register pressure.
+ static void foldPredicateMerge(VPlan &Plan, VPCostContext &CostCtx);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 03a61eff45a51..0a563ec7e3c27 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -204,10 +204,10 @@ define void @nested_mask_two_reductions(ptr %p, ptr %q, i64 %n, ptr %out) {
; IF-EVL-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP5]], <vscale x 2 x i1> [[TMP4]], i32 [[TMP1]])
; IF-EVL-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 2 x i32> [[VP_OP_LOAD2]], zeroinitializer
; IF-EVL-NEXT: [[TMP7:%.*]] = select <vscale x 2 x i1> [[TMP4]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
-; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 2 x i64> [[VEC_PHI]], [[VEC_IND]]
; IF-EVL-NEXT: [[TMP10:%.*]] = mul <vscale x 2 x i64> [[VEC_IND]], [[VEC_IND]]
-; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP10]]
+; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 2 x i64> [[VEC_PHI]], [[VEC_IND]]
; IF-EVL-NEXT: [[TMP12]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI]], i32 [[TMP1]])
+; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP10]]
; IF-EVL-NEXT: [[TMP13]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[VEC_PHI1]], i32 [[TMP1]])
; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[INDEX]]
; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
More information about the llvm-commits
mailing list