[llvm] [LV] Reduce register pressure of RISC-V outer loop reduction. (PR #208621)
Elvis Wang via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 9 00:40:20 PDT 2026
https://github.com/ElvisWang123 updated https://github.com/llvm/llvm-project/pull/208621
>From c75762d2e92febfc6823aa0173f844766175bb53 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Mon, 6 Jul 2026 23:25:43 -0700
Subject: [PATCH 1/2] Precommit tests.
---
.../LoopVectorize/RISCV/reg-usage.ll | 65 +++++++++++++++++++
1 file changed, 65 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 31ccd43911139..86652c8f02a75 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -126,3 +126,68 @@ for.body:
%exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
}
+
+
+define void @red(ptr %base.0, ptr %base.1, ptr %base.2, ptr %base.3, i64 %end) {
+; CHECK-LABEL: 'red'
+; CHECK: LV(REG): Calculating max register usage:
+; CHECK: LV(REG): VF = 1
+; CHECK: LV(REG): Found max usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 10 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): Calculating max register usage:
+; CHECK: LV(REG): VF = vscale x 1
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): VF = vscale x 2
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 16 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): VF = vscale x 4
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 32 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): VF = vscale x 8
+; CHECK: LV(REG): Found max usage: 2 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 64 registers
+; CHECK: LV(REG): Found invariant usage: 1 item
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK: LV(REG): Cost of 64 from 32 spills of RISCV::VRRC
+;
+entry:
+ br label %loop.body
+
+loop.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.body ]
+ %red.0 = phi i64 [ 0, %entry ], [ %red.0.next, %loop.body ]
+ %red.1 = phi i64 [ 0, %entry ], [ %red.1.next, %loop.body ]
+ %red.2 = phi i64 [ 0, %entry ], [ %red.2.next, %loop.body ]
+ %red.3 = phi i64 [ 0, %entry ], [ %red.3.next, %loop.body ]
+ %ptr.0 = getelementptr i8, ptr %base.0, i64 %iv
+ %0 = load i64, ptr %ptr.0, align 8
+ %ptr.1 = getelementptr i8, ptr %base.1, i64 %iv
+ %1 = load i64, ptr %ptr.1, align 8
+ %red.0.next = tail call i64 @llvm.smin.i64(i64 %0, i64 %red.0)
+ %red.2.next = tail call i64 @llvm.smax.i64(i64 %0, i64 %red.2)
+ %red.1.next = tail call i64 @llvm.smin.i64(i64 %1, i64 %red.1)
+ %red.3.next = tail call i64 @llvm.smax.i64(i64 %1, i64 %red.3)
+ %iv.next = add i64 %iv, 1
+ %exitcond = icmp eq i64 %iv, %end
+ br i1 %exitcond, label %loop.exit, label %loop.body
+
+loop.exit:
+ store i64 %red.0.next, ptr %base.0
+ store i64 %red.1.next, ptr %base.1
+ store i64 %red.2.next, ptr %base.2
+ store i64 %red.3.next, ptr %base.3
+ ret void
+}
>From 92883d712cd17e850103bd6db4cc815a6bd30130 Mon Sep 17 00:00:00 2001
From: Elvis Wang <elvis.wang at sifive.com>
Date: Sun, 26 Jul 2026 23:50:02 -0700
Subject: [PATCH 2/2] [LV] Warp inloop operation and predication to reduce reg
pressure.
This patch warp the in-loop operation and the predication (vp.merge)
into one ExpressionRecipe to reduce the register pressure since the
vp.merge can be optmized out.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 6 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 56 ++++++++++++++++++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 42 ++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 4 ++
.../RISCV/reg-usage-maxbandwidth.ll | 47 +++++++++++++---
.../LoopVectorize/RISCV/reg-usage.ll | 18 +++---
.../RISCV/tail-folding-reduction-cost.ll | 12 ++--
.../RISCV/vplan-vp-intrinsics-reduction.ll | 3 +-
.../VPlan/vplan-print-before-after-all.ll | 1 +
10 files changed, 163 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e027dfa834759..066596bce35ab 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6488,6 +6488,7 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
useActiveLaneMask(Style),
useActiveLaneMaskForControlFlow(Style));
+ RUN_VPLAN_PASS(VPlanTransforms::prepareForCostModel, *Plan);
RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
VPlans.push_back(std::move(Plan));
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f37a750a4d516..dfc66271a854f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3615,6 +3615,9 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
/// extended vector operands, negating the multiplication, performing a
/// reduction.add on the result, and adding the scalar result to a chain.
ExtNegatedMulAccReduction,
+ /// Represent an inloop operations with tail-floded select for the outer
+ /// loop reduction.
+ TailFoldedInLoopOp,
};
/// Type of the expression.
@@ -3672,6 +3675,9 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
} else
assert(Neg->getOpcode() == Instruction::FNeg && "Unexpected opcode");
}
+ VPExpressionRecipe(VPSingleDefRecipe *InLoopOp, VPSingleDefRecipe *VPMerge)
+ : VPExpressionRecipe(ExpressionTypes::TailFoldedInLoopOp,
+ {InLoopOp, VPMerge}) {}
~VPExpressionRecipe() override {
SmallPtrSet<VPSingleDefRecipe *, 4> ExpressionRecipesSeen;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7d671e6cd29f1..079e04fad8c0c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3596,9 +3596,11 @@ VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
: VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {},
- cast<VPReductionRecipe>(ExpressionRecipes.back())
- ->getChainOp()
- ->getScalarType()),
+ ExpressionType == ExpressionTypes::TailFoldedInLoopOp
+ ? ExpressionRecipes.back()->getScalarType()
+ : cast<VPReductionRecipe>(ExpressionRecipes.back())
+ ->getChainOp()
+ ->getScalarType()),
ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
assert(!ExpressionRecipes.empty() && "Nothing to combine?");
assert(
@@ -3671,6 +3673,14 @@ SmallVector<VPSingleDefRecipe *> VPExpressionRecipe::decompose() {
InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
VPCostContext &Ctx) const {
+ // Handle expression recipe without in-loop reduction.
+ if (ExpressionType == ExpressionTypes::TailFoldedInLoopOp) {
+ InstructionCost Cost = 0;
+ for (auto *R : ExpressionRecipes)
+ Cost += R->cost(VF, Ctx);
+ return Cost;
+ }
+
Type *RedTy = this->getScalarType();
auto *SrcVecTy =
cast<VectorType>(toVectorTy(getOperand(0)->getScalarType(), VF));
@@ -3742,6 +3752,8 @@ InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
Instruction::ZExt,
Opcode, RedTy, SrcVecTy, Ctx.CostKind);
}
+ case ExpressionTypes::TailFoldedInLoopOp:
+ llvm_unreachable("TailFoldedInLoopOp should be handled early");
}
llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
}
@@ -3772,6 +3784,42 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "EXPRESSION ";
printAsOperand(O, SlotTracker);
O << " = ";
+ // Handle the tail-folded in-loop operation
+ if (ExpressionType == ExpressionTypes::TailFoldedInLoopOp) {
+ VPSingleDefRecipe *InLoopOp = ExpressionRecipes[0];
+ unsigned NumInLoopOps = InLoopOp->getNumOperands();
+ O << "vp.merge ";
+ getOperand(NumInLoopOps)->printAsOperand(O, SlotTracker);
+ O << ", ";
+ auto PrintInLoopOperands = [&]() {
+ O << "(";
+ for (unsigned Idx = 0; Idx != NumInLoopOps; ++Idx) {
+ if (Idx != 0)
+ O << ", ";
+ getOperand(Idx)->printAsOperand(O, SlotTracker);
+ }
+ O << ")";
+ };
+
+ if (auto *WidenIntrinsic = dyn_cast<VPWidenIntrinsicRecipe>(InLoopOp)) {
+ O << WidenIntrinsic->getIntrinsicName();
+ WidenIntrinsic->printFlags(O);
+ PrintInLoopOperands();
+ } else if (auto *Widen = dyn_cast<VPWidenRecipe>(InLoopOp)) {
+ O << Instruction::getOpcodeName(Widen->getOpcode());
+ Widen->printFlags(O);
+ PrintInLoopOperands();
+ } else {
+ llvm_unreachable("Unsupported in-loop recipe for tail-folded expression");
+ }
+ O << ", ";
+ getOperand(NumInLoopOps + 1)->printAsOperand(O, SlotTracker);
+ O << ", ";
+ getOperand(NumInLoopOps + 2)->printAsOperand(O, SlotTracker);
+
+ return;
+ }
+
auto *Red = cast<VPReductionRecipe>(ExpressionRecipes.back());
unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
VPValue *Mask = getOperand(getNumOperands() - 1);
@@ -3866,6 +3914,8 @@ void VPExpressionRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << ")";
break;
}
+ default:
+ llvm_unreachable("Unhandled VPExpressionRecipe::ExpressionTypes enum");
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 63011d01ad9ce..f3da5680115b7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5980,3 +5980,45 @@ void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
}
}
}
+
+static void tryToCreateAbstractInLoopOp(VPWidenIntrinsicRecipe *VPMerge) {
+ // Only convert the in-loop operations with tail-folding predication to
+ // expression recipe.
+ if (any_of(VPMerge->users(), [](VPUser *U) {
+ if (auto *R = dyn_cast<VPSingleDefRecipe>(U))
+ return !isa<VPReductionPHIRecipe>(R) &&
+ !match(R, m_ComputeReductionResult(m_VPValue()));
+ return true;
+ }))
+ return;
+
+ VPSingleDefRecipe *InLoopOp =
+ dyn_cast<VPSingleDefRecipe>(VPMerge->getOperand(1));
+ // TODO: Support cmp-select reductions.
+ if (!InLoopOp || !isa<VPWidenRecipe, VPWidenIntrinsicRecipe>(InLoopOp) ||
+ match(InLoopOp, m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
+ return;
+
+ VPBasicBlock *VPBB = VPMerge->getParent();
+ auto IP = std::next(VPMerge->getIterator());
+ auto *Expr = new VPExpressionRecipe(InLoopOp, VPMerge);
+ Expr->insertBefore(*VPBB, IP);
+ VPMerge->replaceAllUsesWith(Expr);
+}
+
+void VPlanTransforms::prepareForCostModel(VPlan &Plan) {
+ if (Plan.hasScalarVFOnly())
+ return;
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ // Warp in-loop operations and the vp.merge (cleanup tail poison) to
+ // expression recipes since the vp.merge will be optmized out in the
+ // backend.
+ if (match(&R, m_Intrinsic<Intrinsic::vp_merge>(m_VPValue(), m_VPValue(),
+ m_VPValue(), m_VPValue())))
+ tryToCreateAbstractInLoopOp(cast<VPWidenIntrinsicRecipe>(&R));
+ }
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index fef0c14415091..0f9da2585105b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -622,6 +622,10 @@ struct VPlanTransforms {
static void makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx);
+
+ /// Warp recipes in \p Plan into expression recipes that can help
+ /// cost/register pressure estimation.
+ static void prepareForCostModel(VPlan &Plan);
};
} // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
index 41f1a9b45b7ff..e02f8e452f387 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
@@ -3,13 +3,46 @@
; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-maximize-bandwidth -debug-only=loop-vectorize,vplan -disable-output -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
define i32 @dotp(ptr %a, ptr %b) {
-; CHECK-REGS-VP: LV(REG): VF = vscale x 16
-; CHECK-REGS-VP-NEXT: LV(REG): Found max usage: 2 item
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
-; CHECK-REGS-VP-NEXT: LV(REG): Found invariant usage: 1 item
-; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK-REGS-VP: LV: Selecting VF: vscale x 16.
+; CHECK-REGS-VP-LABEL: 'dotp'
+; CHECK-REGS-VP: LV(REG): Calculating max register usage:
+; CHECK-REGS-VP: LV(REG): VF = 1
+; CHECK-REGS-VP: LV(REG): Found max usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): Calculating max register usage:
+; CHECK-REGS-VP: LV(REG): VF = vscale x 1
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 3 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 2
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 3 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 4
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 8
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 12 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV(REG): VF = vscale x 16
+; CHECK-REGS-VP: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; CHECK-REGS-VP: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP: LV: Selecting VF: vscale x 16.
+;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 86652c8f02a75..4259fdd4f35d6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -139,29 +139,29 @@ define void @red(ptr %base.0, ptr %base.1, ptr %base.2, ptr %base.3, i64 %end) {
; CHECK: LV(REG): Calculating max register usage:
; CHECK: LV(REG): VF = vscale x 1
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 8 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 7 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK: LV(REG): VF = vscale x 2
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 16 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 13 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK: LV(REG): VF = vscale x 4
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 32 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 26 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK: LV(REG): VF = vscale x 8
; CHECK: LV(REG): Found max usage: 2 item
-; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 5 registers
-; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 64 registers
+; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 4 registers
+; CHECK: LV(REG): RegisterClass: RISCV::VRRC, 52 registers
; CHECK: LV(REG): Found invariant usage: 1 item
; CHECK: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
-; CHECK: LV(REG): Cost of 64 from 32 spills of RISCV::VRRC
+; CHECK: LV(REG): Cost of 40 from 20 spills of RISCV::VRRC
;
entry:
br label %loop.body
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
index 1c644aadfd8ad..3615797170da1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*(WIDEN-REDUCTION-PHI|WIDEN-INTRINSIC)" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of.*(WIDEN-REDUCTION-PHI|EXPRESSION)" --version 6
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize --disable-output \
; RUN: -tail-folding-policy=prefer-fold-tail -vectorizer-maximize-bandwidth \
@@ -8,15 +8,15 @@
define i64 @add(ptr %a, i64 %n, i64 %start) {
; CHECK-LABEL: 'add'
; CHECK: Cost of 0 for VF vscale x 1: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2:%[0-9]+]]>, vp<[[VP9:%[0-9]+]]>
-; CHECK: Cost of 0 for VF vscale x 1: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 1 for VF vscale x 1: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 2: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 0 for VF vscale x 2: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 2 for VF vscale x 2: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 4: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 0 for VF vscale x 4: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 4 for VF vscale x 4: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 8: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 0 for VF vscale x 8: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 8 for VF vscale x 8: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
; CHECK: Cost of 0 for VF vscale x 16: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP2]]>, vp<[[VP9]]>
-; CHECK: Cost of 16 for VF vscale x 16: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; CHECK: Cost of 32 for VF vscale x 16: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%zext>, ir<%rdx>), ir<%rdx>, vp<%evl>
;
entry:
br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
index 4b84e9bcb687f..590fecddfd2ab 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -91,8 +91,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP7]]>
; IF-EVL-OUTLOOP-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds i32, ir<%arrayidx>, ir<1>
; IF-EVL-OUTLOOP-NEXT: WIDEN ir<%0> = vp.load vp<[[VP8]]>, vp<%evl>
-; IF-EVL-OUTLOOP-NEXT: WIDEN ir<%add> = add ir<%0>, ir<%rdx>
-; IF-EVL-OUTLOOP-NEXT: WIDEN-INTRINSIC vp<[[VP9]]> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%evl>)
+; IF-EVL-OUTLOOP-NEXT: EXPRESSION vp<[[VP9]]> = vp.merge ir<true>, add (ir<%0>, ir<%rdx>), ir<%rdx>, vp<%evl>
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = zext vp<%evl> to i64
; IF-EVL-OUTLOOP-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP10]]>, vp<[[VP5]]>
; IF-EVL-OUTLOOP-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP10]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index c2bd046efae87..9b926dc656986 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -66,6 +66,7 @@
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::narrowInterleaveGroups
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeHeaderMask
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::prepareForCostModel
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printOptimizedVPlan
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMinimumIterationCheck
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceWideCanonicalIVWithWideIV
More information about the llvm-commits
mailing list