[llvm] [RISCV][VPlan] Cost vp.merges that are likely to be folded away as free (PR #209387)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 23:50:06 PDT 2026
https://github.com/lukel97 created https://github.com/llvm/llvm-project/pull/209387
A vp.merge is lowered to a PseudoVMERGE_VVM on RISC-V, and RISCVVectorPeehole usually folds away PseudoVMERGE_VVMs when its true operand is a binary op.
This accounts for this in the cost model and plumbs it through the loop vectorizer. The following reduction loop was previously considered unprofitable to vectorize, but after this change is now vectorized, and folds away all its vp.merges:
void three_gathers(const long *a, const long *c, const long *d,
const long *b, long n, long *out) {
long s1 = 0, s2 = 0, s3 = 0;
for (long i = 0; i < n; i++) {
long j = b[i];
s1 += a[j];
s2 += c[j];
s3 += d[j];
}
out[0] = s1;
out[1] = s2;
out[2] = s3;
}
Some vp.merges have only phis for their operands which can't be folded away, coming from FindLastIV reductions. So this extends and reuses the VectorInstrContext hint to limit the discount when one of the operands is a binary op.
Also see the commit message in 9e77f59005917e32f09136fa43018f471267f5bd which mentions the vp.merge cost should be updated.
>From adb12107728da6b3ef2b8104b5dd65555de30776 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 14 Jul 2026 14:41:41 +0800
Subject: [PATCH] [RISCV][VPlan] Cost vp.merges that are likely to be folded
away as free
A vp.merge is lowered to a PseudoVMERGE_VVM on RISC-V, and RISCVVectorPeehole usually folds away PseudoVMERGE_VVMs when its true operand is a binary op.
This accounts for this in the cost model and plumbs it through the loop vectorizer. The following reduction loop was previously considered unprofitable to vectorize, but after this change is now vectorized, and folds away all its vp.merges:
void three_gathers(const long *a, const long *c, const long *d,
const long *b, long n, long *out) {
long s1 = 0, s2 = 0, s3 = 0;
for (long i = 0; i < n; i++) {
long j = b[i];
s1 += a[j];
s2 += c[j];
s3 += d[j];
}
out[0] = s1;
out[1] = s2;
out[2] = s3;
}
Some vp.merges have only phis for their operands which can't be folded away, coming from FindLastIV reductions. So this extends and reuses the VectorInstrContext hint to limit the discount when one of the operands is a binary op.
Also see the commit message in 9e77f59005917e32f09136fa43018f471267f5bd which mentions the vp.merge cost should be updated.
---
.../llvm/Analysis/TargetTransformInfo.h | 38 +++++++++++--------
llvm/lib/Analysis/TargetTransformInfo.cpp | 13 +++----
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 8 ++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 10 ++++-
.../LoopVectorize/RISCV/force-vect-msg.ll | 2 +-
.../RISCV/tail-folding-reduction-cost.ll | 25 ++++++++----
6 files changed, 64 insertions(+), 32 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 10c0509460b95..8f14d6e353cd7 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -175,6 +175,24 @@ class MemIntrinsicCostAttributes {
Align getAlignment() const { return Alignment; }
};
+/// Represents a hint about the context in which a vector instruction or
+/// intrinsic is used.
+///
+/// On some targets, inserts/extracts can cheaply be folded into loads/stores.
+/// Similarly, vp.merge can also be folded into binary ops on some targets.s
+///
+/// This enum allows the vectorizer to give getVectorInstrCost and
+/// getIntrinsicInstrCost an idea of how the values are used.
+///
+/// See \c getVectorInstrContextHint to compute a VectorInstrContext from an
+/// insert/extract Instruction*.
+enum class VectorInstrContext : uint8_t {
+ None, ///< The instruction is not folded.
+ Load, ///< The value being inserted comes from a load (InsertElement only).
+ Store, ///< The extracted value is stored (ExtractElement only).
+ BinaryOp, ///< One of the operands is a binary op.
+};
+
class IntrinsicCostAttributes {
const IntrinsicInst *II = nullptr;
Type *RetTy = nullptr;
@@ -185,6 +203,7 @@ class IntrinsicCostAttributes {
// If ScalarizationCost is UINT_MAX, the cost of scalarizing the
// arguments and the return value will be computed based on types.
InstructionCost ScalarizationCost = InstructionCost::getInvalid();
+ VectorInstrContext VIC = VectorInstrContext::None;
public:
LLVM_ABI IntrinsicCostAttributes(
@@ -204,13 +223,15 @@ class IntrinsicCostAttributes {
Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
ArrayRef<Type *> Tys, FastMathFlags Flags = FastMathFlags(),
const IntrinsicInst *I = nullptr,
- InstructionCost ScalarCost = InstructionCost::getInvalid());
+ InstructionCost ScalarCost = InstructionCost::getInvalid(),
+ VectorInstrContext VIC = VectorInstrContext::None);
Intrinsic::ID getID() const { return IID; }
const IntrinsicInst *getInst() const { return II; }
Type *getReturnType() const { return RetTy; }
FastMathFlags getFlags() const { return FMF; }
InstructionCost getScalarizationCost() const { return ScalarizationCost; }
+ VectorInstrContext getVectorInstrContext() const { return VIC; }
const SmallVectorImpl<const Value *> &getArgs() const { return Arguments; }
const SmallVectorImpl<Type *> &getArgTypes() const { return ParamTys; }
@@ -1056,20 +1077,7 @@ class TargetTransformInfo {
isTargetIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID,
int RetIdx) const;
- /// Represents a hint about the context in which an insert/extract is used.
- ///
- /// On some targets, inserts/extracts can cheaply be folded into loads/stores.
- ///
- /// This enum allows the vectorizer to give getVectorInstrCost an idea of how
- /// inserts/extracts are used
- ///
- /// See \c getVectorInstrContextHint to compute a VectorInstrContext from an
- /// insert/extract Instruction*.
- enum class VectorInstrContext : uint8_t {
- None, ///< The insert/extract is not used with a load/store.
- Load, ///< The value being inserted comes from a load (InsertElement only).
- Store, ///< The extracted value is stored (ExtractElement only).
- };
+ using VectorInstrContext = llvm::VectorInstrContext;
/// Calculates a VectorInstrContext from \p I.
LLVM_ABI static VectorInstrContext
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index cdccd04f4c9ea..fcaf07de034ab 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -106,13 +106,12 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
ParamTys.push_back(Argument->getType());
}
-IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
- ArrayRef<const Value *> Args,
- ArrayRef<Type *> Tys,
- FastMathFlags Flags,
- const IntrinsicInst *I,
- InstructionCost ScalarCost)
- : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
+IntrinsicCostAttributes::IntrinsicCostAttributes(
+ Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
+ ArrayRef<Type *> Tys, FastMathFlags Flags, const IntrinsicInst *I,
+ InstructionCost ScalarCost, VectorInstrContext VIC)
+ : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost),
+ VIC(VIC) {
ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index a50c1664ad5a1..47030266549ef 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1720,6 +1720,14 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
cast<VectorType>(ICA.getArgTypes()[0]), {}, CostKind,
0, cast<VectorType>(ICA.getReturnType()));
}
+ case Intrinsic::vp_merge: {
+ // If an operand is a binary op and the type is legal, RISCVVectorPeephole
+ // will likely fold the resulting vmerge.vvm away.
+ if (ICA.getVectorInstrContext() == VectorInstrContext::BinaryOp &&
+ getTypeLegalizationCost(RetTy).first == 1)
+ return TTI::TCC_Free;
+ break;
+ }
case Intrinsic::fptoui_sat:
case Intrinsic::fptosi_sat: {
InstructionCost Cost = 0;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 3fbeb7e772a2e..c62c4a082b95e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2319,11 +2319,19 @@ InstructionCost VPWidenIntrinsicRecipe::computeCallCost(
return toVectorTy(Op->getScalarType(), VF);
});
+ VectorInstrContext VIC = VectorInstrContext::None;
+ for (const VPValue *Op : Operands)
+ if (isa<VPWidenRecipe>(Op) &&
+ Instruction::isBinaryOp(cast<VPWidenRecipe>(Op)->getOpcode())) {
+ VIC = VectorInstrContext::BinaryOp;
+ break;
+ }
+
// TODO: Rework TTI interface to avoid reliance on underlying IntrinsicInst.
IntrinsicCostAttributes CostAttrs(
ID, RetTy, Arguments, ParamTys, R.getFastMathFlagsOrNone(),
dyn_cast_or_null<IntrinsicInst>(R.getUnderlyingValue()),
- InstructionCost::getInvalid());
+ InstructionCost::getInvalid(), VIC);
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
index f4be7b5490d7f..7adb8b6e1b2cf 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
@@ -4,7 +4,7 @@
; CHECK: LV: Loop hints: force=enabled
; CHECK: LV: Scalar loop costs: 4.
; ChosenFactor.Cost is 11, but the real cost will be divided by the width, which is 2.8
-; CHECK: Cost for VF vscale x 2: 11
+; CHECK: Cost for VF vscale x 2: 9
; Regardless of force vectorization or not, this loop will eventually be vectorized because of the cost model.
; Therefore, the following message does not need to be printed even if vectorization is explicitly forced in the metadata.
; CHECK-NOT: LV: Vectorization seems to be not beneficial, but was forced by a user.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
index 7ed53097cf12f..99445ce6c0d04 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction-cost.ll
@@ -1,25 +1,34 @@
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize --disable-output \
-; RUN: -tail-folding-policy=prefer-fold-tail \
+; RUN: -tail-folding-policy=prefer-fold-tail -vectorizer-maximize-bandwidth \
; RUN: -mtriple=riscv64 -mattr=+v -S < %s 2>&1 | FileCheck %s
; CHECK: Cost of 0 for VF vscale x 4: WIDEN-REDUCTION-PHI ir<%rdx> = phi
-; CHECK: Cost of 2 for VF vscale x 4: WIDEN-INTRINSIC vp<%{{.+}}> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%{{.+}}>)
+; CHECK: Cost of 0 for VF vscale x 4: WIDEN-INTRINSIC vp<%{{.+}}> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%{{.+}}>)
-define i32 @add(ptr %a, i64 %n, i32 %start) {
+; CHECK: Cost of 0 for VF vscale x 8: WIDEN-REDUCTION-PHI ir<%rdx> = phi
+; CHECK: Cost of 0 for VF vscale x 8: WIDEN-INTRINSIC vp<%{{.+}}> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%{{.+}}>)
+
+; Type needs split, won't be folded:
+
+; CHECK: Cost of 0 for VF vscale x 16: WIDEN-REDUCTION-PHI ir<%rdx> = phi
+; CHECK: Cost of 16 for VF vscale x 16: WIDEN-INTRINSIC vp<%{{.+}}> = call llvm.vp.merge(ir<true>, ir<%add>, ir<%rdx>, vp<%{{.+}}>)
+
+define i64 @add(ptr %a, i64 %n, i64 %start) {
entry:
br label %loop
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %rdx = phi i32 [ %start, %entry ], [ %add, %loop ]
- %arrayidx = getelementptr inbounds i32, ptr %a, i64 %iv
- %0 = load i32, ptr %arrayidx, align 4
- %add = add nsw i32 %0, %rdx
+ %rdx = phi i64 [ %start, %entry ], [ %add, %loop ]
+ %arrayidx = getelementptr inbounds i8, ptr %a, i64 %iv
+ %0 = load i8, ptr %arrayidx
+ %zext = zext i8 %0 to i64
+ %add = add nsw i64 %zext, %rdx
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
br i1 %exitcond.not, label %exit, label %loop
exit:
- ret i32 %add
+ ret i64 %add
}
More information about the llvm-commits
mailing list