[llvm] [VPlan] Embed widening decisions in recipes (NFCI) (PR #195385)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Sat May 2 06:17:59 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/195385
>From f0d6d76ab5c6fc38a8e01cd1102b0def756c3205 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 29 Apr 2026 14:54:04 +0100
Subject: [PATCH] [VPlan] Embed widening decisions in recipes
We currently make widening decisions in an ad-hoc fashion, and have
helpers that unnecessarily do recursive reasoning over and over:
isSingleScalar, onlyFirstLaneUsed, and onlyScalarValuesUsed. The goal is
to make widening decisions in a principled fashion, simply by analyzing
the Plan, and this patch is the first step towards the goal. By
embedding widening decisions in recipes, which can be refined
iteratively, we can eliminate the hard-coded wide-equivalents of
VPInstruction, including VPWidenRecipe, VPWidenGEPRecipe, and
VPWidenCastRecipe. The design we have picked would also allow us to
eliminate VPReplicateRecipe.
This patch is intentionally non-functional, but it should have enough
information to see what the follow-ups would be.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 165 +++++++++++-------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 88 ++++++++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 79 +++++----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 7 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 59 +------
.../Transforms/Vectorize/VPlanTest.cpp | 3 +-
7 files changed, 231 insertions(+), 172 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e912751525fc7..e74726d90d690 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7848,7 +7848,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
[](const VPUser *U) {
return isa<VPScalarIVStepsRecipe>(U) ||
isa<VPDerivedIVRecipe>(U) ||
- cast<VPRecipeBase>(U)->isScalarCast() ||
+ cast<VPInstruction>(U)->isScalarCast() ||
cast<VPInstruction>(U)->getOpcode() ==
Instruction::Add;
}) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 592d83af2295f..171aee90974e5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -409,6 +409,9 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Subclass identifier (for isa/dyn_cast).
const unsigned char SubclassID;
+ /// Three bits of widening information, which takes values in VPWideningTy.
+ unsigned char WideningInfo : 3;
+
/// Each VPRecipe belongs to a single VPBasicBlock.
VPBasicBlock *Parent = nullptr;
@@ -466,9 +469,29 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPLastPHISC = VPReductionPHISC,
};
- VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ /// An enumeration for keeping track of whether the widening status of the
+ /// recipe.
+ /// If only the Narrow bit is set, it means that the recipe necessarily emits
+ /// a narrow scalar result.
+ /// If only the Wide bit is set, it means that the recipe necessarily emits a
+ /// widened result.
+ /// If only the ReplicatePart bit is set, it means that the recipe necessarily
+ /// emits scalar results for each part when unrolling.
+ /// If Wide & Narrow are set, it indicates a Wide recipe with a Narrow opcode,
+ /// that could be narrowed if profitable.
+ /// If ReplicatePart & Narrow are set, it indicates a ReplicatePart recipe
+ /// with a Narrow opcode, that could be narrowed if profitable.
+ using VPWideningTy = enum {
+ Narrow = 1 << 0,
+ Wide = 1 << 1,
+ ReplicatePart = 1 << 2,
+ };
+
+ VPRecipeBase(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPDef(), VPUser(Operands), SubclassID(SC), DL(DL) {}
+ : VPDef(), VPUser(Operands), SubclassID(SC), WideningInfo(WideInfo),
+ DL(DL) {}
~VPRecipeBase() override = default;
@@ -552,8 +575,13 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Returns the debug location of the recipe.
DebugLoc getDebugLoc() const { return DL; }
- /// Return true if the recipe is a scalar cast.
- bool isScalarCast() const;
+ /// Methods for querying and setting WideningInfo.
+ bool isNarrow() const { return WideningInfo == Narrow; }
+ void markNarrow() { WideningInfo = Narrow; }
+ void markPossiblyNarrow() { WideningInfo |= Narrow; }
+ void markReplicatePart() { WideningInfo = ReplicatePart; }
+ bool maybeNarrow() const { return WideningInfo & Narrow; }
+ bool maybeReplicatePart() const { return WideningInfo & ReplicatePart; }
/// Set the recipe's debug location to \p NewDL.
void setDebugLoc(DebugLoc NewDL) { DL = NewDL; }
@@ -604,13 +632,15 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Note that VPRecipeBase must be inherited from before VPValue.
class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
public:
- VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ VPSingleDefRecipe(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
+ : VPRecipeBase(SC, WideInfo, Operands, DL), VPRecipeValue(this) {}
- VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
- Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
+ VPSingleDefRecipe(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands, Value *UV,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeBase(SC, WideInfo, Operands, DL), VPRecipeValue(this, UV) {}
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
@@ -1107,10 +1137,10 @@ static_assert(sizeof(VPIRFlags) <= 3, "VPIRFlags should not grow");
/// A pure-virtual common base class for recipes defining a single VPValue and
/// using IR flags.
struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- const VPIRFlags &Flags,
+ VPRecipeWithIRFlags(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands, const VPIRFlags &Flags,
DebugLoc DL = DebugLoc::getUnknown())
- : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
+ : VPSingleDefRecipe(SC, WideInfo, Operands, DL), VPIRFlags(Flags) {}
static inline bool classof(const VPRecipeBase *R) {
return R->getVPRecipeID() == VPRecipeBase::VPBlendSC ||
@@ -1489,6 +1519,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// result is also a single scalar.
bool isSingleScalar() const;
+ /// Returns true if this VPInstruction is a scalar cast.
+ bool isScalarCast() const { return Instruction::isCast(getOpcode()); }
+
/// Returns the symbolic name assigned to the VPInstruction.
StringRef getName() const { return Name; }
@@ -1524,8 +1557,6 @@ class VPInstructionWithType : public VPInstruction {
static inline bool classof(const VPRecipeBase *R) {
// VPInstructionWithType are VPInstructions with specific opcodes requiring
// type information.
- if (R->isScalarCast())
- return true;
auto *VPI = dyn_cast<VPInstruction>(R);
if (!VPI)
return false;
@@ -1536,7 +1567,7 @@ class VPInstructionWithType : public VPInstruction {
case Instruction::Load:
return true;
default:
- return false;
+ return VPI->isScalarCast();
}
}
@@ -1684,7 +1715,8 @@ class VPIRInstruction : public VPRecipeBase {
/// VPIRInstruction::create() should be used to create VPIRInstructions, as
/// subclasses may need to be created, e.g. VPIRPhi.
VPIRInstruction(Instruction &I)
- : VPRecipeBase(VPRecipeBase::VPIRInstructionSC, {}), I(I) {}
+ : VPRecipeBase(VPRecipeBase::VPIRInstructionSC, VPRecipeBase::Narrow, {}),
+ I(I) {}
public:
~VPIRInstruction() override = default;
@@ -1779,17 +1811,11 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
public:
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
- DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
- VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
- setUnderlyingValue(&I);
- }
+ DebugLoc DL = {});
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
- DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
- VPIRMetadata(Metadata), Opcode(Opcode) {}
+ DebugLoc DL = {});
~VPWidenRecipe() override = default;
@@ -1841,7 +1867,8 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, VPRecipeBase::Wide, Op,
+ Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
@@ -1903,8 +1930,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
- Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC,
+ VPRecipeBase::Wide, CallArguments, Flags, DL),
VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
@@ -1917,8 +1944,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
- Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC,
+ VPRecipeBase::Wide, CallArguments, Flags, DL),
VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
@@ -1992,8 +2019,8 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
ArrayRef<VPValue *> CallArguments,
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, CallArguments, Flags,
- DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, VPRecipeBase::Wide,
+ CallArguments, Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(
@@ -2044,7 +2071,8 @@ class VPHistogramRecipe : public VPRecipeBase {
public:
VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(VPRecipeBase::VPHistogramSC, Operands, DL),
+ : VPRecipeBase(VPRecipeBase::VPHistogramSC, VPRecipeBase::Wide, Operands,
+ DL),
Opcode(Opcode) {}
~VPHistogramRecipe() override = default;
@@ -2094,7 +2122,9 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC,
+ VPRecipeBase::Wide | VPRecipeBase::Narrow, Operands,
+ Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
@@ -2153,8 +2183,8 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
public:
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *SourceElementTy,
int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC, {Ptr, VF},
- GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC,
+ VPRecipeBase::Narrow, {Ptr, VF}, GEPFlags, DL),
SourceElementTy(SourceElementTy), Stride(Stride) {
assert(Stride < 0 && "Stride must be negative");
}
@@ -2222,7 +2252,8 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
public:
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr, GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC,
+ VPRecipeBase::Narrow, Ptr, GEPFlags, DL),
SourceElementTy(SourceElementTy) {}
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorPointerSC)
@@ -2296,7 +2327,8 @@ class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe,
protected:
VPHeaderPHIRecipe(unsigned char VPRecipeID, Instruction *UnderlyingInstr,
VPValue *Start, DebugLoc DL = DebugLoc::getUnknown())
- : VPSingleDefRecipe(VPRecipeID, Start, UnderlyingInstr, DL) {}
+ : VPSingleDefRecipe(VPRecipeID, VPRecipeBase::Wide, Start,
+ UnderlyingInstr, DL) {}
const VPRecipeBase *getAsRecipe() const override { return this; }
@@ -2585,7 +2617,8 @@ class LLVM_ABI_FOR_TEST VPWidenPHIRecipe : public VPSingleDefRecipe,
/// debug location \p DL and \p Name.
VPWidenPHIRecipe(ArrayRef<VPValue *> IncomingValues,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues, DL),
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, VPRecipeBase::Wide,
+ IncomingValues, DL),
Name(Name.str()) {}
VPWidenPHIRecipe *clone() override {
@@ -2780,7 +2813,9 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
/// all other incoming values are merged into it.
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPBlendSC, Operands, Flags, DL) {
+ : VPRecipeWithIRFlags(VPRecipeBase::VPBlendSC,
+ VPRecipeBase::Wide | VPRecipeBase::Narrow, Operands,
+ Flags, DL) {
assert(Operands.size() >= 2 && "Expected at least two operands!");
setUnderlyingValue(Phi);
}
@@ -2861,8 +2896,8 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
ArrayRef<VPValue *> Operands,
ArrayRef<VPValue *> StoredValues, VPValue *Mask,
bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPIRMetadata(MD), IG(IG),
- NeedsMaskForGaps(NeedsMaskForGaps) {
+ : VPRecipeBase(SC, VPRecipeBase::Wide, Operands, DL), VPIRMetadata(MD),
+ IG(IG), NeedsMaskForGaps(NeedsMaskForGaps) {
// TODO: extend the masked interleaved-group support to reversed access.
assert((!Mask || !IG->isReverse()) &&
"Reversed masked interleave-group not supported.");
@@ -3046,13 +3081,15 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
FastMathFlags FMFs, Instruction *I,
ArrayRef<VPValue *> Operands, VPValue *CondOp,
ReductionStyle Style, DebugLoc DL)
- : VPRecipeWithIRFlags(SC, Operands, FMFs, DL), RdxKind(RdxKind),
- Style(Style) {
+ : VPRecipeWithIRFlags(SC, VPRecipeBase::Wide, Operands, FMFs, DL),
+ RdxKind(RdxKind), Style(Style) {
if (CondOp) {
IsConditional = true;
addOperand(CondOp);
}
setUnderlyingValue(I);
+ if (!isPartialReduction())
+ markNarrow();
}
public:
@@ -3188,9 +3225,6 @@ class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
/// a single scalar, only one copy will be generated.
class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
- /// Indicator if only a single replica per lane is needed.
- bool IsSingleScalar;
-
/// Indicator if the replicas are also predicated.
bool IsPredicated;
@@ -3198,20 +3232,13 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
bool IsSingleScalar, VPValue *Mask = nullptr,
const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
- DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPReplicateSC, Operands, Flags, DL),
- VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
- IsPredicated(Mask) {
- setUnderlyingValue(I);
- if (Mask)
- addOperand(Mask);
- }
+ DebugLoc DL = DebugLoc::getUnknown());
~VPReplicateRecipe() override = default;
VPReplicateRecipe *clone() override {
auto *Copy = new VPReplicateRecipe(
- getUnderlyingInstr(), operands(), IsSingleScalar,
+ getUnderlyingInstr(), operands(), isNarrow(),
isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc());
Copy->transferFlags(*this);
return Copy;
@@ -3228,7 +3255,8 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- bool isSingleScalar() const { return IsSingleScalar; }
+ /// TODO: Remove.
+ bool isSingleScalar() const { return isNarrow(); }
bool isPredicated() const { return IsPredicated; }
@@ -3271,7 +3299,8 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase {
public:
VPBranchOnMaskRecipe(VPValue *BlockInMask, DebugLoc DL)
- : VPRecipeBase(VPRecipeBase::VPBranchOnMaskSC, {BlockInMask}, DL) {}
+ : VPRecipeBase(VPRecipeBase::VPBranchOnMaskSC, VPRecipeBase::Narrow,
+ {BlockInMask}, DL) {}
VPBranchOnMaskRecipe *clone() override {
return new VPBranchOnMaskRecipe(getOperand(0), getDebugLoc());
@@ -3459,8 +3488,10 @@ class LLVM_ABI_FOR_TEST VPPredInstPHIRecipe : public VPSingleDefRecipe {
public:
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask.
+ /// FIXME: The WideInfo should include Narrow.
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
- : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV, DL) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, VPRecipeBase::Wide,
+ PredV, DL) {}
~VPPredInstPHIRecipe() override = default;
VPPredInstPHIRecipe *clone() override {
@@ -3523,7 +3554,8 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
std::initializer_list<VPValue *> Operands,
bool Consecutive, const VPIRMetadata &Metadata,
DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
+ : VPRecipeBase(SC, VPRecipeBase::Wide, Operands, DL),
+ VPIRMetadata(Metadata), Ingredient(I),
Alignment(getLoadStoreAlignment(&I)), Consecutive(Consecutive) {}
public:
@@ -3753,7 +3785,9 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
public:
VPExpandSCEVRecipe(const SCEV *Expr)
- : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}), Expr(Expr) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, VPRecipeBase::Narrow,
+ {}),
+ Expr(Expr) {}
~VPExpandSCEVRecipe() override = default;
@@ -3862,7 +3896,8 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
public VPUnrollPartAccessor<1> {
public:
VPWidenCanonicalIVRecipe(VPRegionValue *CanonicalIV)
- : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC, {CanonicalIV}) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC,
+ VPRecipeBase::Wide, {CanonicalIV}) {}
~VPWidenCanonicalIVRecipe() override = default;
@@ -3918,7 +3953,8 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
const FPMathOperator *FPBinOp, VPIRValue *Start,
VPValue *IV, VPValue *Step)
- : VPSingleDefRecipe(VPRecipeBase::VPDerivedIVSC, {Start, IV, Step}),
+ : VPSingleDefRecipe(VPRecipeBase::VPDerivedIVSC, VPRecipeBase::Narrow,
+ {Start, IV, Step}),
Kind(Kind), FPBinOp(FPBinOp) {}
~VPDerivedIVRecipe() override = default;
@@ -3978,8 +4014,9 @@ class LLVM_ABI_FOR_TEST VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step, VPValue *VF,
Instruction::BinaryOps Opcode, FastMathFlags FMFs,
DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPScalarIVStepsSC, {IV, Step, VF},
- FMFs, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPScalarIVStepsSC,
+ VPRecipeBase::ReplicatePart, {IV, Step, VF}, FMFs,
+ DL),
InductionOpcode(Opcode) {}
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ef217b92640c4..4d4709f418bd0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -312,11 +312,6 @@ bool VPRecipeBase::isPhi() const {
isa<VPPhi, VPIRPhi>(this);
}
-bool VPRecipeBase::isScalarCast() const {
- auto *VPI = dyn_cast<VPInstruction>(this);
- return VPI && Instruction::isCast(VPI->getOpcode());
-}
-
void VPIRFlags::intersectFlags(const VPIRFlags &Other) {
assert(OpType == Other.OpType && "OpType must match");
switch (OpType) {
@@ -423,11 +418,39 @@ template class VPUnrollPartAccessor<2>;
template class VPUnrollPartAccessor<3>;
}
+/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
+/// uniform, the result will also be uniform.
+static bool possiblyNarrowOpcode(unsigned Opcode) {
+ if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
+ return true;
+ switch (Opcode) {
+ case Instruction::Freeze:
+ case Instruction::GetElementPtr:
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ case Instruction::Select:
+ case VPInstruction::Not:
+ case VPInstruction::Broadcast:
+ case VPInstruction::MaskedCond:
+ case VPInstruction::PtrAdd:
+ return true;
+ default:
+ return false;
+ }
+}
+
VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &MD,
DebugLoc DL, const Twine &Name)
- : VPRecipeWithIRFlags(VPRecipeBase::VPInstructionSC, Operands, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPInstructionSC, VPRecipeBase::Wide,
+ Operands, Flags, DL),
VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {
+ if (isSingleScalar() || isVectorToScalar())
+ markNarrow();
+ else if (is_contained({VPInstruction::Unpack, VPInstruction::PtrAdd}, Opcode))
+ markReplicatePart();
+ if (possiblyNarrowOpcode(Opcode))
+ markPossiblyNarrow();
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
assert(hasRequiredFlagsForOpcode(getOpcode()) &&
@@ -526,7 +549,8 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
}
bool VPInstruction::doesGeneratePerAllLanes() const {
- return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
+ return Opcode == VPInstruction::Unpack ||
+ (Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this));
}
bool VPInstruction::canGenerateScalarForFirstLane() const {
@@ -2313,6 +2337,27 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
}
#endif
+VPWidenRecipe::VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags,
+ const VPIRMetadata &Metadata, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, VPRecipeBase::Wide, Operands,
+ Flags, DL),
+ VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
+ setUnderlyingValue(&I);
+ if (possiblyNarrowOpcode(Opcode))
+ markPossiblyNarrow();
+}
+
+VPWidenRecipe::VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags,
+ const VPIRMetadata &Metadata, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, VPRecipeBase::Wide, Operands,
+ Flags, DL),
+ VPIRMetadata(Metadata), Opcode(Opcode) {
+ if (possiblyNarrowOpcode(Opcode))
+ markPossiblyNarrow();
+}
+
void VPWidenRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
switch (Opcode) {
@@ -2945,7 +2990,8 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
- : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {}, {}),
+ : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, VPRecipeBase::Wide, {},
+ {}),
ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
assert(!ExpressionRecipes.empty() && "Nothing to combine?");
assert(
@@ -2998,6 +3044,9 @@ VPExpressionRecipe::VPExpressionRecipe(
for (auto *R : ExpressionRecipes)
for (auto const &[LiveIn, Tmp] : zip(operands(), LiveInPlaceholders))
R->replaceUsesOfWith(LiveIn, Tmp);
+
+ if (isSingleScalar())
+ markNarrow();
}
void VPExpressionRecipe::decompose() {
@@ -3296,12 +3345,29 @@ static void scalarizeInstruction(const Instruction *Instr,
"are defined outside the vectorized region.");
}
+VPReplicateRecipe::VPReplicateRecipe(Instruction *I,
+ ArrayRef<VPValue *> Operands,
+ bool IsSingleScalar, VPValue *Mask,
+ const VPIRFlags &Flags,
+ VPIRMetadata Metadata, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPReplicateSC,
+ IsSingleScalar ? VPRecipeBase::Narrow
+ : VPRecipeBase::ReplicatePart,
+ Operands, Flags, DL),
+ VPIRMetadata(Metadata), IsPredicated(Mask) {
+ setUnderlyingValue(I);
+ if (Mask)
+ addOperand(Mask);
+ if (possiblyNarrowOpcode(I->getOpcode()))
+ markPossiblyNarrow();
+}
+
void VPReplicateRecipe::execute(VPTransformState &State) {
Instruction *UI = getUnderlyingInstr();
if (!State.Lane) {
- assert(IsSingleScalar && "VPReplicateRecipes outside replicate regions "
- "must have already been unrolled");
+ assert(isSingleScalar() && "VPReplicateRecipes outside replicate regions "
+ "must have already been unrolled");
scalarizeInstruction(UI, this, VPLane(0), State);
return;
}
@@ -3657,7 +3723,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPReplicateRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
- O << Indent << (IsSingleScalar ? "CLONE " : "REPLICATE ");
+ O << Indent << (isSingleScalar() ? "CLONE " : "REPLICATE ");
if (!getUnderlyingInstr()->getType()->isVoidTy()) {
printAsOperand(O, SlotTracker);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 11e5b18bca871..fbbb664db3908 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1855,34 +1855,37 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe>(&R))
- continue;
- auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
- if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
- continue;
-
- auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(&R);
- if (RepR && RepR->getOpcode() == Instruction::Store &&
- vputils::isSingleScalar(RepR->getOperand(1))) {
- auto *Clone = new VPReplicateRecipe(
- RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
- true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
- *RepR /*Metadata*/, RepR->getDebugLoc());
- Clone->insertBefore(RepOrWidenR);
- VPBuilder Builder(Clone);
- VPValue *ExtractOp = Clone->getOperand(0);
- if (vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1)))
+ if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
+ if (RepR->isPredicated())
+ continue;
+ if (RepR->getOpcode() == Instruction::Store &&
+ vputils::isSingleScalar(RepR->getOperand(1))) {
+ auto *Clone = new VPReplicateRecipe(
+ RepR->getUnderlyingInstr(), RepR->operands(),
+ true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
+ *RepR /*Metadata*/, RepR->getDebugLoc());
+ Clone->insertBefore(RepR);
+ VPBuilder Builder(Clone);
+ VPValue *ExtractOp = Clone->getOperand(0);
+ if (vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1)))
+ ExtractOp =
+ Builder.createNaryOp(VPInstruction::ExtractLastPart, ExtractOp);
ExtractOp =
- Builder.createNaryOp(VPInstruction::ExtractLastPart, ExtractOp);
- ExtractOp =
- Builder.createNaryOp(VPInstruction::ExtractLastLane, ExtractOp);
- Clone->setOperand(0, ExtractOp);
- RepR->eraseFromParent();
+ Builder.createNaryOp(VPInstruction::ExtractLastLane, ExtractOp);
+ Clone->setOperand(0, ExtractOp);
+ RepR->eraseFromParent();
+ } else if (vputils::isSingleScalar(RepR))
+ RepR->markNarrow();
continue;
}
+ // To narrow Widens, we need to check profitability beyond isSingleScalar.
+ if (!isa<VPWidenRecipe, VPWidenGEPRecipe>(&R))
+ continue;
+ auto *WidenR = cast<VPRecipeWithIRFlags>(&R);
+
// Skip recipes that aren't single scalars.
- if (!vputils::isSingleScalar(RepOrWidenR))
+ if (!vputils::isSingleScalar(WidenR))
continue;
// Predicate to check if a user of Op introduces extra broadcasts.
@@ -1899,11 +1902,10 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
};
};
- if (any_of(RepOrWidenR->users(), IntroducesBCastOf(RepOrWidenR)) &&
- none_of(RepOrWidenR->operands(), [&](VPValue *Op) {
- if (any_of(
- make_filter_range(Op->users(), not_equal_to(RepOrWidenR)),
- IntroducesBCastOf(Op)))
+ if (any_of(WidenR->users(), IntroducesBCastOf(WidenR)) &&
+ none_of(WidenR->operands(), [&](VPValue *Op) {
+ if (any_of(make_filter_range(Op->users(), not_equal_to(WidenR)),
+ IntroducesBCastOf(Op)))
return false;
// Non-constant live-ins require broadcasts, while constants do not
// need explicit broadcasts.
@@ -1915,12 +1917,12 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
continue;
auto *Clone = new VPReplicateRecipe(
- RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
- true /*IsSingleScalar*/, nullptr, *RepOrWidenR);
- Clone->insertBefore(RepOrWidenR);
- RepOrWidenR->replaceAllUsesWith(Clone);
- if (isDeadRecipe(*RepOrWidenR))
- RepOrWidenR->eraseFromParent();
+ WidenR->getUnderlyingInstr(), WidenR->operands(),
+ true /*IsSingleScalar*/, nullptr, *WidenR);
+ Clone->insertBefore(WidenR);
+ WidenR->replaceAllUsesWith(Clone);
+ if (isDeadRecipe(*WidenR))
+ WidenR->eraseFromParent();
}
}
}
@@ -4895,18 +4897,15 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
for (VPBasicBlock *VPBB :
concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPScalarIVStepsRecipe, VPReplicateRecipe, VPInstruction>(&R))
+ if (!R.maybeReplicatePart())
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
auto UsesVectorOrInsideReplicateRegion = [DefR, LoopRegion](VPUser *U) {
VPRegionBlock *ParentRegion = cast<VPRecipeBase>(U)->getRegion();
return !U->usesScalars(DefR) || ParentRegion != LoopRegion;
};
- if ((isa<VPReplicateRecipe>(DefR) &&
- cast<VPReplicateRecipe>(DefR)->isSingleScalar()) ||
- (isa<VPInstruction>(DefR) &&
- (vputils::onlyFirstLaneUsed(DefR) ||
- !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes())) ||
+ if ((isa<VPInstruction>(DefR) &&
+ !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes()) ||
none_of(DefR->users(), UsesVectorOrInsideReplicateRegion))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index d5eb7e82b6f84..8adeb3ef4d7c6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -862,12 +862,9 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
SmallVector<VPRecipeBase *> ToRemove;
for (VPBasicBlock *VPBB : VPBBsToUnroll) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPInstruction, VPReplicateRecipe, VPScalarIVStepsRecipe>(&R) ||
- (isa<VPReplicateRecipe>(&R) &&
- cast<VPReplicateRecipe>(&R)->isSingleScalar()) ||
+ if (!R.maybeReplicatePart() ||
(isa<VPInstruction>(&R) &&
- !cast<VPInstruction>(&R)->doesGeneratePerAllLanes() &&
- cast<VPInstruction>(&R)->getOpcode() != VPInstruction::Unpack))
+ !cast<VPInstruction>(&R)->doesGeneratePerAllLanes()))
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index a60b490a69ce6..66cd627bd1e19 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -338,62 +338,25 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
match(Addr, m_scev_AffineAddRec(m_SCEV(), m_SCEV()));
}
-/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
-/// uniform, the result will also be uniform.
-static bool preservesUniformity(unsigned Opcode) {
- if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
- return true;
- switch (Opcode) {
- case Instruction::Freeze:
- case Instruction::GetElementPtr:
- case Instruction::ICmp:
- case Instruction::FCmp:
- case Instruction::Select:
- case VPInstruction::Not:
- case VPInstruction::Broadcast:
- case VPInstruction::MaskedCond:
- case VPInstruction::PtrAdd:
- return true;
- default:
- return false;
- }
-}
-
bool vputils::isSingleScalar(const VPValue *VPV) {
// Live-in, symbolic and region-values represent single-scalar values.
if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
return true;
- if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
+ // Use WideningInfo present directly on the recipe.
+ const VPRecipeBase *R = VPV->getDefiningRecipe();
+ if (R->isNarrow())
+ return true;
+
+ if (auto *Rep = dyn_cast<VPReplicateRecipe>(R)) {
const VPRegionBlock *RegionOfR = Rep->getRegion();
// Don't consider recipes in replicate regions as uniform yet; their first
// lane cannot be accessed when executing the replicate region for other
// lanes.
if (RegionOfR && RegionOfR->isReplicator())
return false;
- return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
- all_of(Rep->operands(), isSingleScalar));
- }
- if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
- return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
- if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
- return preservesUniformity(WidenR->getOpcode()) &&
- all_of(WidenR->operands(), isSingleScalar);
}
- if (auto *VPI = dyn_cast<VPInstruction>(VPV))
- return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- (preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isSingleScalar));
- if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
- return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
- return true;
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
- return Expr->isSingleScalar();
-
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ return R->maybeNarrow() && all_of(R->operands(), isSingleScalar);
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -424,14 +387,10 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPWidenRecipe *R) {
- return preservesUniformity(R->getOpcode()) &&
+ .Case<VPWidenRecipe, VPInstruction>([](const auto *R) {
+ return R->maybeNarrow() &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPInstruction *VPI) {
- return preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isUniformAcrossVFsAndUFs);
- })
.Case([](const VPWidenCastRecipe *R) {
// A cast is uniform according to its operand.
return isUniformAcrossVFsAndUFs(R->getOperand(0));
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index a1ddda7eda969..dd9c1332d54e2 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1697,7 +1697,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
} // namespace
struct VPDoubleValueDef : public VPRecipeBase {
- VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
+ VPDoubleValueDef(ArrayRef<VPValue *> Operands)
+ : VPRecipeBase(99, VPRecipeBase::Wide, Operands) {
new VPRecipeValue(this);
new VPRecipeValue(this);
}
More information about the llvm-commits
mailing list