[llvm] [VPlan] Embed widening decisions in recipes (NFCI) (PR #195385)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Sun May 3 09:01:51 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/195385
>From fcd35e67cff14cb40cf576268acd65606fbaece4 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 29 Apr 2026 14:54:04 +0100
Subject: [PATCH 1/4] [VPlan] Embed widening decisions in recipes
We currently make widening decisions in an ad-hoc fashion, and have
helpers that unnecessarily do recursive reasoning over and over:
isSingleScalar, onlyFirstLaneUsed, and onlyScalarValuesUsed. The goal is
to make widening decisions in a principled fashion, simply by analyzing
the Plan, and this patch is the first step towards the goal. By
embedding widening decisions in recipes, which can be refined
iteratively, we can eliminate the hard-coded wide-equivalents of
VPInstruction, including VPWidenRecipe, VPWidenGEPRecipe, and
VPWidenCastRecipe. The design we have picked would also allow us to
eliminate VPReplicateRecipe.
This patch is intentionally non-functional, but it should have enough
information to see what the follow-ups would be.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 176 ++++++++++--------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 104 +++++++++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 113 ++++++-----
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 7 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 61 +-----
.../Transforms/Vectorize/VPlanTest.cpp | 3 +-
7 files changed, 258 insertions(+), 208 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 78163b5fe35d5..f1bb40a592def 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7845,7 +7845,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
[](const VPUser *U) {
return isa<VPScalarIVStepsRecipe>(U) ||
isa<VPDerivedIVRecipe>(U) ||
- cast<VPRecipeBase>(U)->isScalarCast() ||
+ cast<VPInstruction>(U)->isScalarCast() ||
cast<VPInstruction>(U)->getOpcode() ==
Instruction::Add;
}) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 592d83af2295f..824dac1fd07f2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -409,6 +409,9 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Subclass identifier (for isa/dyn_cast).
const unsigned char SubclassID;
+ /// Three bits of widening information, which takes values in VPWideningTy.
+ unsigned char WideningInfo : 3;
+
/// Each VPRecipe belongs to a single VPBasicBlock.
VPBasicBlock *Parent = nullptr;
@@ -466,9 +469,23 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPLastPHISC = VPReductionPHISC,
};
- VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ /// An enumeration for keeping track of the widening status of the recipe.
+ /// The recipe necessarily produceds a scalar value if only the Narrow bit is
+ /// set, a wide value if only the Wide bit is set, and scalar values for each
+ /// unroll part if only the ReplicatePart bit is set. The Narrow bit can be
+ /// set on Wide and ReplicatePart recipes, which indicates that the recipe
+ /// could be considered narrow if profitable.
+ using VPWideningTy = enum {
+ Narrow = 1 << 0,
+ Wide = 1 << 1,
+ ReplicatePart = 1 << 2,
+ };
+
+ VPRecipeBase(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPDef(), VPUser(Operands), SubclassID(SC), DL(DL) {}
+ : VPDef(), VPUser(Operands), SubclassID(SC), WideningInfo(WideInfo),
+ DL(DL) {}
~VPRecipeBase() override = default;
@@ -552,8 +569,13 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Returns the debug location of the recipe.
DebugLoc getDebugLoc() const { return DL; }
- /// Return true if the recipe is a scalar cast.
- bool isScalarCast() const;
+ /// Methods for querying and setting WideningInfo.
+ bool isNarrow() const { return WideningInfo == Narrow; }
+ void markNarrow() { WideningInfo = Narrow; }
+ void markPossiblyNarrow() { WideningInfo |= Narrow; }
+ void markReplicatePart() { WideningInfo = ReplicatePart; }
+ bool maybeNarrow() const { return WideningInfo & Narrow; }
+ bool maybeReplicatePart() const { return WideningInfo & ReplicatePart; }
/// Set the recipe's debug location to \p NewDL.
void setDebugLoc(DebugLoc NewDL) { DL = NewDL; }
@@ -604,13 +626,15 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Note that VPRecipeBase must be inherited from before VPValue.
class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
public:
- VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ VPSingleDefRecipe(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
+ : VPRecipeBase(SC, WideInfo, Operands, DL), VPRecipeValue(this) {}
- VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
- Value *UV, DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
+ VPSingleDefRecipe(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands, Value *UV,
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeBase(SC, WideInfo, Operands, DL), VPRecipeValue(this, UV) {}
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
@@ -1107,10 +1131,10 @@ static_assert(sizeof(VPIRFlags) <= 3, "VPIRFlags should not grow");
/// A pure-virtual common base class for recipes defining a single VPValue and
/// using IR flags.
struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- const VPIRFlags &Flags,
+ VPRecipeWithIRFlags(const unsigned char SC, unsigned char WideInfo,
+ ArrayRef<VPValue *> Operands, const VPIRFlags &Flags,
DebugLoc DL = DebugLoc::getUnknown())
- : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
+ : VPSingleDefRecipe(SC, WideInfo, Operands, DL), VPIRFlags(Flags) {}
static inline bool classof(const VPRecipeBase *R) {
return R->getVPRecipeID() == VPRecipeBase::VPBlendSC ||
@@ -1379,6 +1403,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
return Opcode == Instruction::PHI || Opcode == Instruction::GetElementPtr;
}
+ /// Returns true if this VPInstruction produces a scalar value from a vector,
+ /// e.g. by performing a reduction or extracting a lane.
+ bool isVectorToScalar() const;
+
public:
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
@@ -1481,13 +1509,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// Returns true if the recipe only uses the first part of operand \p Op.
bool usesFirstPartOnly(const VPValue *Op) const override;
- /// Returns true if this VPInstruction produces a scalar value from a vector,
- /// e.g. by performing a reduction or extracting a lane.
- bool isVectorToScalar() const;
-
- /// Returns true if this VPInstruction's operands are single scalars and the
- /// result is also a single scalar.
- bool isSingleScalar() const;
+ /// Returns true if this VPInstruction is a scalar cast.
+ bool isScalarCast() const { return Instruction::isCast(getOpcode()); }
/// Returns the symbolic name assigned to the VPInstruction.
StringRef getName() const { return Name; }
@@ -1496,6 +1519,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
void setName(StringRef NewName) { Name = NewName.str(); }
protected:
+ /// Returns true if this VPInstruction's operands are single scalars and the
+ /// result is also a single scalar.
+ bool isSingleScalar() const;
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the VPInstruction to \p O.
void printRecipe(raw_ostream &O, const Twine &Indent,
@@ -1524,8 +1551,6 @@ class VPInstructionWithType : public VPInstruction {
static inline bool classof(const VPRecipeBase *R) {
// VPInstructionWithType are VPInstructions with specific opcodes requiring
// type information.
- if (R->isScalarCast())
- return true;
auto *VPI = dyn_cast<VPInstruction>(R);
if (!VPI)
return false;
@@ -1536,7 +1561,7 @@ class VPInstructionWithType : public VPInstruction {
case Instruction::Load:
return true;
default:
- return false;
+ return VPI->isScalarCast();
}
}
@@ -1684,7 +1709,8 @@ class VPIRInstruction : public VPRecipeBase {
/// VPIRInstruction::create() should be used to create VPIRInstructions, as
/// subclasses may need to be created, e.g. VPIRPhi.
VPIRInstruction(Instruction &I)
- : VPRecipeBase(VPRecipeBase::VPIRInstructionSC, {}), I(I) {}
+ : VPRecipeBase(VPRecipeBase::VPIRInstructionSC, VPRecipeBase::Narrow, {}),
+ I(I) {}
public:
~VPIRInstruction() override = default;
@@ -1779,17 +1805,11 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
public:
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
- DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
- VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
- setUnderlyingValue(&I);
- }
+ DebugLoc DL = {});
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
- DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
- VPIRMetadata(Metadata), Opcode(Opcode) {}
+ DebugLoc DL = {});
~VPWidenRecipe() override = default;
@@ -1841,7 +1861,8 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, VPRecipeBase::Wide, Op,
+ Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
@@ -1903,8 +1924,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
- Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC,
+ VPRecipeBase::Wide, CallArguments, Flags, DL),
VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
@@ -1917,8 +1938,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
- Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC,
+ VPRecipeBase::Wide, CallArguments, Flags, DL),
VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
ResultTy(Ty) {
LLVMContext &Ctx = Ty->getContext();
@@ -1992,8 +2013,8 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
ArrayRef<VPValue *> CallArguments,
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, CallArguments, Flags,
- DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, VPRecipeBase::Wide,
+ CallArguments, Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(
@@ -2044,7 +2065,8 @@ class VPHistogramRecipe : public VPRecipeBase {
public:
VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeBase(VPRecipeBase::VPHistogramSC, Operands, DL),
+ : VPRecipeBase(VPRecipeBase::VPHistogramSC, VPRecipeBase::Wide, Operands,
+ DL),
Opcode(Opcode) {}
~VPHistogramRecipe() override = default;
@@ -2094,7 +2116,9 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC,
+ VPRecipeBase::Wide | VPRecipeBase::Narrow, Operands,
+ Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
@@ -2153,8 +2177,8 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags {
public:
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *SourceElementTy,
int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC, {Ptr, VF},
- GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC,
+ VPRecipeBase::Narrow, {Ptr, VF}, GEPFlags, DL),
SourceElementTy(SourceElementTy), Stride(Stride) {
assert(Stride < 0 && "Stride must be negative");
}
@@ -2222,7 +2246,8 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
public:
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
GEPNoWrapFlags GEPFlags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr, GEPFlags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC,
+ VPRecipeBase::Narrow, Ptr, GEPFlags, DL),
SourceElementTy(SourceElementTy) {}
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorPointerSC)
@@ -2296,7 +2321,8 @@ class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe,
protected:
VPHeaderPHIRecipe(unsigned char VPRecipeID, Instruction *UnderlyingInstr,
VPValue *Start, DebugLoc DL = DebugLoc::getUnknown())
- : VPSingleDefRecipe(VPRecipeID, Start, UnderlyingInstr, DL) {}
+ : VPSingleDefRecipe(VPRecipeID, VPRecipeBase::Wide, Start,
+ UnderlyingInstr, DL) {}
const VPRecipeBase *getAsRecipe() const override { return this; }
@@ -2585,7 +2611,8 @@ class LLVM_ABI_FOR_TEST VPWidenPHIRecipe : public VPSingleDefRecipe,
/// debug location \p DL and \p Name.
VPWidenPHIRecipe(ArrayRef<VPValue *> IncomingValues,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, IncomingValues, DL),
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, VPRecipeBase::Wide,
+ IncomingValues, DL),
Name(Name.str()) {}
VPWidenPHIRecipe *clone() override {
@@ -2780,7 +2807,9 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
/// all other incoming values are merged into it.
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPBlendSC, Operands, Flags, DL) {
+ : VPRecipeWithIRFlags(VPRecipeBase::VPBlendSC,
+ VPRecipeBase::Wide | VPRecipeBase::Narrow, Operands,
+ Flags, DL) {
assert(Operands.size() >= 2 && "Expected at least two operands!");
setUnderlyingValue(Phi);
}
@@ -2861,8 +2890,8 @@ class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
ArrayRef<VPValue *> Operands,
ArrayRef<VPValue *> StoredValues, VPValue *Mask,
bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPIRMetadata(MD), IG(IG),
- NeedsMaskForGaps(NeedsMaskForGaps) {
+ : VPRecipeBase(SC, VPRecipeBase::Wide, Operands, DL), VPIRMetadata(MD),
+ IG(IG), NeedsMaskForGaps(NeedsMaskForGaps) {
// TODO: extend the masked interleaved-group support to reversed access.
assert((!Mask || !IG->isReverse()) &&
"Reversed masked interleave-group not supported.");
@@ -3046,13 +3075,15 @@ class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
FastMathFlags FMFs, Instruction *I,
ArrayRef<VPValue *> Operands, VPValue *CondOp,
ReductionStyle Style, DebugLoc DL)
- : VPRecipeWithIRFlags(SC, Operands, FMFs, DL), RdxKind(RdxKind),
- Style(Style) {
+ : VPRecipeWithIRFlags(SC, VPRecipeBase::Wide, Operands, FMFs, DL),
+ RdxKind(RdxKind), Style(Style) {
if (CondOp) {
IsConditional = true;
addOperand(CondOp);
}
setUnderlyingValue(I);
+ if (!isPartialReduction())
+ markNarrow();
}
public:
@@ -3188,9 +3219,6 @@ class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
/// a single scalar, only one copy will be generated.
class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
- /// Indicator if only a single replica per lane is needed.
- bool IsSingleScalar;
-
/// Indicator if the replicas are also predicated.
bool IsPredicated;
@@ -3198,20 +3226,13 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
bool IsSingleScalar, VPValue *Mask = nullptr,
const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
- DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPRecipeBase::VPReplicateSC, Operands, Flags, DL),
- VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
- IsPredicated(Mask) {
- setUnderlyingValue(I);
- if (Mask)
- addOperand(Mask);
- }
+ DebugLoc DL = DebugLoc::getUnknown());
~VPReplicateRecipe() override = default;
VPReplicateRecipe *clone() override {
auto *Copy = new VPReplicateRecipe(
- getUnderlyingInstr(), operands(), IsSingleScalar,
+ getUnderlyingInstr(), operands(), isNarrow(),
isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc());
Copy->transferFlags(*this);
return Copy;
@@ -3228,15 +3249,13 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override;
- bool isSingleScalar() const { return IsSingleScalar; }
-
bool isPredicated() const { return IsPredicated; }
/// Returns true if the recipe only uses the first lane of operand \p Op.
bool usesFirstLaneOnly(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
"Op must be an operand of the recipe");
- return isSingleScalar();
+ return isNarrow();
}
/// Returns true if the recipe uses scalars of operand \p Op.
@@ -3271,7 +3290,8 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase {
public:
VPBranchOnMaskRecipe(VPValue *BlockInMask, DebugLoc DL)
- : VPRecipeBase(VPRecipeBase::VPBranchOnMaskSC, {BlockInMask}, DL) {}
+ : VPRecipeBase(VPRecipeBase::VPBranchOnMaskSC, VPRecipeBase::Narrow,
+ {BlockInMask}, DL) {}
VPBranchOnMaskRecipe *clone() override {
return new VPBranchOnMaskRecipe(getOperand(0), getDebugLoc());
@@ -3439,10 +3459,10 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
/// effects.
bool mayHaveSideEffects() const;
+protected:
/// Returns true if the result of this VPExpressionRecipe is a single-scalar.
bool isSingleScalar() const;
-protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void printRecipe(raw_ostream &O, const Twine &Indent,
@@ -3460,7 +3480,9 @@ class LLVM_ABI_FOR_TEST VPPredInstPHIRecipe : public VPSingleDefRecipe {
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask.
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
- : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV, DL) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC,
+ VPRecipeBase::Wide | VPRecipeBase::Narrow, PredV,
+ DL) {}
~VPPredInstPHIRecipe() override = default;
VPPredInstPHIRecipe *clone() override {
@@ -3523,7 +3545,8 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
std::initializer_list<VPValue *> Operands,
bool Consecutive, const VPIRMetadata &Metadata,
DebugLoc DL)
- : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
+ : VPRecipeBase(SC, VPRecipeBase::Wide, Operands, DL),
+ VPIRMetadata(Metadata), Ingredient(I),
Alignment(getLoadStoreAlignment(&I)), Consecutive(Consecutive) {}
public:
@@ -3753,7 +3776,9 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
public:
VPExpandSCEVRecipe(const SCEV *Expr)
- : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}), Expr(Expr) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, VPRecipeBase::Narrow,
+ {}),
+ Expr(Expr) {}
~VPExpandSCEVRecipe() override = default;
@@ -3862,7 +3887,8 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
public VPUnrollPartAccessor<1> {
public:
VPWidenCanonicalIVRecipe(VPRegionValue *CanonicalIV)
- : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC, {CanonicalIV}) {}
+ : VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC,
+ VPRecipeBase::Wide, {CanonicalIV}) {}
~VPWidenCanonicalIVRecipe() override = default;
@@ -3918,7 +3944,8 @@ class VPDerivedIVRecipe : public VPSingleDefRecipe {
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
const FPMathOperator *FPBinOp, VPIRValue *Start,
VPValue *IV, VPValue *Step)
- : VPSingleDefRecipe(VPRecipeBase::VPDerivedIVSC, {Start, IV, Step}),
+ : VPSingleDefRecipe(VPRecipeBase::VPDerivedIVSC, VPRecipeBase::Narrow,
+ {Start, IV, Step}),
Kind(Kind), FPBinOp(FPBinOp) {}
~VPDerivedIVRecipe() override = default;
@@ -3978,8 +4005,9 @@ class LLVM_ABI_FOR_TEST VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step, VPValue *VF,
Instruction::BinaryOps Opcode, FastMathFlags FMFs,
DebugLoc DL)
- : VPRecipeWithIRFlags(VPRecipeBase::VPScalarIVStepsSC, {IV, Step, VF},
- FMFs, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPScalarIVStepsSC,
+ VPRecipeBase::ReplicatePart, {IV, Step, VF}, FMFs,
+ DL),
InductionOpcode(Opcode) {}
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ef217b92640c4..0dcc667808448 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -312,11 +312,6 @@ bool VPRecipeBase::isPhi() const {
isa<VPPhi, VPIRPhi>(this);
}
-bool VPRecipeBase::isScalarCast() const {
- auto *VPI = dyn_cast<VPInstruction>(this);
- return VPI && Instruction::isCast(VPI->getOpcode());
-}
-
void VPIRFlags::intersectFlags(const VPIRFlags &Other) {
assert(OpType == Other.OpType && "OpType must match");
switch (OpType) {
@@ -423,11 +418,39 @@ template class VPUnrollPartAccessor<2>;
template class VPUnrollPartAccessor<3>;
}
+/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
+/// uniform, the result will also be uniform.
+static bool possiblyNarrowOpcode(unsigned Opcode) {
+ if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
+ return true;
+ switch (Opcode) {
+ case Instruction::Freeze:
+ case Instruction::GetElementPtr:
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ case Instruction::Select:
+ case VPInstruction::Not:
+ case VPInstruction::Broadcast:
+ case VPInstruction::MaskedCond:
+ case VPInstruction::PtrAdd:
+ return true;
+ default:
+ return false;
+ }
+}
+
VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &MD,
DebugLoc DL, const Twine &Name)
- : VPRecipeWithIRFlags(VPRecipeBase::VPInstructionSC, Operands, Flags, DL),
+ : VPRecipeWithIRFlags(VPRecipeBase::VPInstructionSC, VPRecipeBase::Wide,
+ Operands, Flags, DL),
VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {
+ if (isSingleScalar() || isVectorToScalar())
+ markNarrow();
+ else if (is_contained({VPInstruction::Unpack, VPInstruction::PtrAdd}, Opcode))
+ markReplicatePart();
+ if (possiblyNarrowOpcode(Opcode))
+ markPossiblyNarrow();
assert(flagsValidForOpcode(getOpcode()) &&
"Set flags not supported for the provided opcode");
assert(hasRequiredFlagsForOpcode(getOpcode()) &&
@@ -526,7 +549,8 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
}
bool VPInstruction::doesGeneratePerAllLanes() const {
- return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
+ return Opcode == VPInstruction::Unpack ||
+ (Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this));
}
bool VPInstruction::canGenerateScalarForFirstLane() const {
@@ -2313,6 +2337,27 @@ void VPIRFlags::printFlags(raw_ostream &O) const {
}
#endif
+VPWidenRecipe::VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags,
+ const VPIRMetadata &Metadata, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, VPRecipeBase::Wide, Operands,
+ Flags, DL),
+ VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
+ setUnderlyingValue(&I);
+ if (possiblyNarrowOpcode(Opcode))
+ markPossiblyNarrow();
+}
+
+VPWidenRecipe::VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags,
+ const VPIRMetadata &Metadata, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, VPRecipeBase::Wide, Operands,
+ Flags, DL),
+ VPIRMetadata(Metadata), Opcode(Opcode) {
+ if (possiblyNarrowOpcode(Opcode))
+ markPossiblyNarrow();
+}
+
void VPWidenRecipe::execute(VPTransformState &State) {
auto &Builder = State.Builder;
switch (Opcode) {
@@ -2945,7 +2990,8 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
- : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, {}, {}),
+ : VPSingleDefRecipe(VPRecipeBase::VPExpressionSC, VPRecipeBase::Wide, {},
+ {}),
ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {
assert(!ExpressionRecipes.empty() && "Nothing to combine?");
assert(
@@ -2998,6 +3044,9 @@ VPExpressionRecipe::VPExpressionRecipe(
for (auto *R : ExpressionRecipes)
for (auto const &[LiveIn, Tmp] : zip(operands(), LiveInPlaceholders))
R->replaceUsesOfWith(LiveIn, Tmp);
+
+ if (isSingleScalar())
+ markNarrow();
}
void VPExpressionRecipe::decompose() {
@@ -3296,17 +3345,34 @@ static void scalarizeInstruction(const Instruction *Instr,
"are defined outside the vectorized region.");
}
+VPReplicateRecipe::VPReplicateRecipe(Instruction *I,
+ ArrayRef<VPValue *> Operands,
+ bool IsSingleScalar, VPValue *Mask,
+ const VPIRFlags &Flags,
+ VPIRMetadata Metadata, DebugLoc DL)
+ : VPRecipeWithIRFlags(VPRecipeBase::VPReplicateSC,
+ IsSingleScalar ? VPRecipeBase::Narrow
+ : VPRecipeBase::ReplicatePart,
+ Operands, Flags, DL),
+ VPIRMetadata(Metadata), IsPredicated(Mask) {
+ setUnderlyingValue(I);
+ if (Mask)
+ addOperand(Mask);
+ if (possiblyNarrowOpcode(I->getOpcode()))
+ markPossiblyNarrow();
+}
+
void VPReplicateRecipe::execute(VPTransformState &State) {
Instruction *UI = getUnderlyingInstr();
if (!State.Lane) {
- assert(IsSingleScalar && "VPReplicateRecipes outside replicate regions "
- "must have already been unrolled");
+ assert(isNarrow() && "VPReplicateRecipes outside replicate regions "
+ "must have already been unrolled");
scalarizeInstruction(UI, this, VPLane(0), State);
return;
}
- assert((State.VF.isScalar() || !isSingleScalar()) &&
+ assert((State.VF.isScalar() || !isNarrow()) &&
"uniform recipe shouldn't be predicated");
assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
scalarizeInstruction(UI, this, *State.Lane, State);
@@ -3417,7 +3483,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
// transform, avoid computing their cost multiple times for now.
Ctx.SkipCostComputation.insert(UI);
- if (VF.isScalable() && !isSingleScalar())
+ if (VF.isScalable() && !isNarrow())
return InstructionCost::getInvalid();
switch (UI->getOpcode()) {
@@ -3463,7 +3529,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
Type *ResultTy = Ctx.Types.inferScalarType(this);
InstructionCost ScalarCallCost =
Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
- if (isSingleScalar()) {
+ if (isNarrow()) {
if (CalledFn->isIntrinsic())
ScalarCallCost = std::min(
ScalarCallCost,
@@ -3493,14 +3559,14 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
case Instruction::FCmp:
return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
Ctx) *
- (isSingleScalar() ? 1 : VF.getFixedValue());
+ (isNarrow() ? 1 : VF.getFixedValue());
case Instruction::SDiv:
case Instruction::UDiv:
case Instruction::SRem:
case Instruction::URem: {
InstructionCost ScalarCost =
getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1), Ctx);
- if (isSingleScalar())
+ if (isNarrow())
return ScalarCost;
// If any of the operands is from a different replicate region and has its
@@ -3575,13 +3641,13 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
return UniformCost;
}
- Type *PtrTy = isSingleScalar() ? ScalarPtrTy : toVectorTy(ScalarPtrTy, VF);
+ Type *PtrTy = isNarrow() ? ScalarPtrTy : toVectorTy(ScalarPtrTy, VF);
InstructionCost ScalarCost =
ScalarMemOpCost +
Ctx.TTI.getAddressComputationCost(
PtrTy, UsedByLoadStoreAddress ? nullptr : Ctx.PSE.getSE(), PtrSCEV,
Ctx.CostKind);
- if (isSingleScalar())
+ if (isNarrow())
return ScalarCost;
SmallVector<const VPValue *> OpsToScalarize;
@@ -3644,7 +3710,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
case Instruction::AddrSpaceCast: {
return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
Ctx) *
- (isSingleScalar() ? 1 : VF.getFixedValue());
+ (isNarrow() ? 1 : VF.getFixedValue());
}
case Instruction::ExtractValue:
case Instruction::InsertValue:
@@ -3657,7 +3723,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPReplicateRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
- O << Indent << (IsSingleScalar ? "CLONE " : "REPLICATE ");
+ O << Indent << (isNarrow() ? "CLONE " : "REPLICATE ");
if (!getUnderlyingInstr()->getType()->isVoidTy()) {
printAsOperand(O, SlotTracker);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9b34bd1f092a0..c851878b3b975 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -310,7 +310,7 @@ static bool sinkScalarOperands(VPlan &Plan) {
return;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(Candidate))
- if (!ScalarVFOnly && RepR->isSingleScalar())
+ if (!ScalarVFOnly && RepR->isNarrow())
return;
WorkList.insert({SinkTo, Candidate});
@@ -520,7 +520,7 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
// mask but in the replicate region.
auto *RecipeWithoutMask = new VPReplicateRecipe(
PredRecipe->getUnderlyingInstr(), drop_end(PredRecipe->operands()),
- PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
+ PredRecipe->isNarrow(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
PredRecipe->getDebugLoc());
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
@@ -908,7 +908,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
// Skip recipes that shouldn't be narrowed.
if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
Def->getNumUsers() == 0 || !Def->getUnderlyingValue() ||
- (RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
+ (RepR && (RepR->isNarrow() || RepR->isPredicated())))
continue;
// Skip recipes that may have other lanes than their first used.
@@ -1857,34 +1857,37 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe>(&R))
- continue;
- auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
- if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
- continue;
-
- auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(&R);
- if (RepR && RepR->getOpcode() == Instruction::Store &&
- vputils::isSingleScalar(RepR->getOperand(1))) {
- auto *Clone = new VPReplicateRecipe(
- RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
- true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
- *RepR /*Metadata*/, RepR->getDebugLoc());
- Clone->insertBefore(RepOrWidenR);
- VPBuilder Builder(Clone);
- VPValue *ExtractOp = Clone->getOperand(0);
- if (vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1)))
+ if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
+ if (RepR->isPredicated())
+ continue;
+ if (RepR->getOpcode() == Instruction::Store &&
+ vputils::isSingleScalar(RepR->getOperand(1))) {
+ auto *Clone = new VPReplicateRecipe(
+ RepR->getUnderlyingInstr(), RepR->operands(),
+ true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
+ *RepR /*Metadata*/, RepR->getDebugLoc());
+ Clone->insertBefore(RepR);
+ VPBuilder Builder(Clone);
+ VPValue *ExtractOp = Clone->getOperand(0);
+ if (vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1)))
+ ExtractOp =
+ Builder.createNaryOp(VPInstruction::ExtractLastPart, ExtractOp);
ExtractOp =
- Builder.createNaryOp(VPInstruction::ExtractLastPart, ExtractOp);
- ExtractOp =
- Builder.createNaryOp(VPInstruction::ExtractLastLane, ExtractOp);
- Clone->setOperand(0, ExtractOp);
- RepR->eraseFromParent();
+ Builder.createNaryOp(VPInstruction::ExtractLastLane, ExtractOp);
+ Clone->setOperand(0, ExtractOp);
+ RepR->eraseFromParent();
+ } else if (vputils::isSingleScalar(RepR))
+ RepR->markNarrow();
continue;
}
+ // To narrow Widens, we need to check profitability beyond isSingleScalar.
+ if (!isa<VPWidenRecipe, VPWidenGEPRecipe>(&R))
+ continue;
+ auto *WidenR = cast<VPRecipeWithIRFlags>(&R);
+
// Skip recipes that aren't single scalars.
- if (!vputils::isSingleScalar(RepOrWidenR))
+ if (!vputils::isSingleScalar(WidenR))
continue;
// Predicate to check if a user of Op introduces extra broadcasts.
@@ -1901,28 +1904,27 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
};
};
- if (any_of(RepOrWidenR->users(), IntroducesBCastOf(RepOrWidenR)) &&
- none_of(RepOrWidenR->operands(), [&](VPValue *Op) {
- if (any_of(
- make_filter_range(Op->users(), not_equal_to(RepOrWidenR)),
- IntroducesBCastOf(Op)))
+ if (any_of(WidenR->users(), IntroducesBCastOf(WidenR)) &&
+ none_of(WidenR->operands(), [&](VPValue *Op) {
+ if (any_of(make_filter_range(Op->users(), not_equal_to(WidenR)),
+ IntroducesBCastOf(Op)))
return false;
// Non-constant live-ins require broadcasts, while constants do not
// need explicit broadcasts.
auto *IRV = dyn_cast<VPIRValue>(Op);
bool LiveInNeedsBroadcast = IRV && !isa<Constant>(IRV->getValue());
auto *OpR = dyn_cast<VPReplicateRecipe>(Op);
- return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
+ return LiveInNeedsBroadcast || (OpR && OpR->isNarrow());
}))
continue;
auto *Clone = new VPReplicateRecipe(
- RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
- true /*IsSingleScalar*/, nullptr, *RepOrWidenR);
- Clone->insertBefore(RepOrWidenR);
- RepOrWidenR->replaceAllUsesWith(Clone);
- if (isDeadRecipe(*RepOrWidenR))
- RepOrWidenR->eraseFromParent();
+ WidenR->getUnderlyingInstr(), WidenR->operands(),
+ true /*IsSingleScalar*/, nullptr, *WidenR);
+ Clone->insertBefore(WidenR);
+ WidenR->replaceAllUsesWith(Clone);
+ if (isDeadRecipe(*WidenR))
+ WidenR->eraseFromParent();
}
}
}
@@ -2524,7 +2526,7 @@ static void licm(VPlan &Plan) {
// replicates to single-scalar replicates.
// TODO: When unrolling, replicateByVF doesn't handle sunk
// non-single-scalar replicates correctly.
- if (!RepR->isSingleScalar())
+ if (!RepR->isNarrow())
continue;
}
@@ -4603,7 +4605,7 @@ void VPlanTransforms::hoistInvariantLoads(VPlan &Plan) {
for (VPRecipeBase &R : *VPBB) {
// Only handle single-scalar replicated loads with invariant addresses.
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
- if (RepR->isPredicated() || !RepR->isSingleScalar() ||
+ if (RepR->isPredicated() || !RepR->isNarrow() ||
RepR->getOpcode() != Instruction::Load)
continue;
@@ -4745,18 +4747,18 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
// Find the load with minimum alignment to use.
auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
- bool IsSingleScalar = EarliestLoad->isSingleScalar();
+ bool IsNarrow = EarliestLoad->isNarrow();
assert(all_of(Group,
- [IsSingleScalar](VPReplicateRecipe *R) {
- return R->isSingleScalar() == IsSingleScalar;
+ [IsNarrow](VPReplicateRecipe *R) {
+ return R->isNarrow() == IsNarrow;
}) &&
- "all members in group must agree on IsSingleScalar");
+ "all members in group must agree on narrowing");
// Create an unpredicated version of the earliest load with common
// metadata.
auto *UnpredicatedLoad = new VPReplicateRecipe(
LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(0)},
- IsSingleScalar, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
+ IsNarrow, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
UnpredicatedLoad->insertBefore(EarliestLoad);
@@ -4813,10 +4815,10 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
VPValue *SelectedValue = Group[0]->getOperand(0);
VPBuilder Builder(InsertBB, LastStore->getIterator());
- bool IsSingleScalar = Group[0]->isSingleScalar();
+ bool IsNarrow = Group[0]->isNarrow();
for (unsigned I = 1; I < Group.size(); ++I) {
- assert(IsSingleScalar == Group[I]->isSingleScalar() &&
- "all members in group must agree on IsSingleScalar");
+ assert(IsNarrow == Group[I]->isNarrow() &&
+ "all members in group must agree on narrowing");
VPValue *Mask = Group[I]->getMask();
VPValue *Value = Group[I]->getOperand(0);
SelectedValue = Builder.createSelect(Mask, Value, SelectedValue,
@@ -4829,7 +4831,7 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
// Create unconditional store with selected value and common metadata.
auto *UnpredicatedStore = new VPReplicateRecipe(
StoreWithMinAlign->getUnderlyingInstr(),
- {SelectedValue, LastStore->getOperand(1)}, IsSingleScalar,
+ {SelectedValue, LastStore->getOperand(1)}, IsNarrow,
/*Mask=*/nullptr, *LastStore, CommonMetadata);
UnpredicatedStore->insertBefore(*InsertBB, LastStore->getIterator());
@@ -4905,18 +4907,15 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
for (VPBasicBlock *VPBB :
concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPScalarIVStepsRecipe, VPReplicateRecipe, VPInstruction>(&R))
+ if (!R.maybeReplicatePart())
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
auto UsesVectorOrInsideReplicateRegion = [DefR, LoopRegion](VPUser *U) {
VPRegionBlock *ParentRegion = cast<VPRecipeBase>(U)->getRegion();
return !U->usesScalars(DefR) || ParentRegion != LoopRegion;
};
- if ((isa<VPReplicateRecipe>(DefR) &&
- cast<VPReplicateRecipe>(DefR)->isSingleScalar()) ||
- (isa<VPInstruction>(DefR) &&
- (vputils::onlyFirstLaneUsed(DefR) ||
- !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes())) ||
+ if ((isa<VPInstruction>(DefR) &&
+ !cast<VPInstruction>(DefR)->doesGeneratePerAllLanes()) ||
none_of(DefR->users(), UsesVectorOrInsideReplicateRegion))
continue;
@@ -5240,7 +5239,7 @@ static bool isAlreadyNarrow(VPValue *VPV) {
if (isa<VPIRValue>(VPV))
return true;
auto *RepR = dyn_cast<VPReplicateRecipe>(VPV);
- return RepR && RepR->isSingleScalar();
+ return RepR && RepR->isNarrow();
}
// Convert a wide recipe defining a VPValue \p V feeding an interleave group to
@@ -5276,7 +5275,7 @@ narrowInterleaveGroupOp(VPValue *V, SmallPtrSetImpl<VPValue *> &NarrowedOps) {
}
if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
- assert(RepR->isSingleScalar() && RepR->getOpcode() == Instruction::Load &&
+ assert(RepR->isNarrow() && RepR->getOpcode() == Instruction::Load &&
"must be a single scalar load");
NarrowedOps.insert(RepR);
return RepR;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index d5eb7e82b6f84..8adeb3ef4d7c6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -862,12 +862,9 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
SmallVector<VPRecipeBase *> ToRemove;
for (VPBasicBlock *VPBB : VPBBsToUnroll) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!isa<VPInstruction, VPReplicateRecipe, VPScalarIVStepsRecipe>(&R) ||
- (isa<VPReplicateRecipe>(&R) &&
- cast<VPReplicateRecipe>(&R)->isSingleScalar()) ||
+ if (!R.maybeReplicatePart() ||
(isa<VPInstruction>(&R) &&
- !cast<VPInstruction>(&R)->doesGeneratePerAllLanes() &&
- cast<VPInstruction>(&R)->getOpcode() != VPInstruction::Unpack))
+ !cast<VPInstruction>(&R)->doesGeneratePerAllLanes()))
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index a60b490a69ce6..8c73b667e960f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -338,62 +338,25 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
match(Addr, m_scev_AffineAddRec(m_SCEV(), m_SCEV()));
}
-/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
-/// uniform, the result will also be uniform.
-static bool preservesUniformity(unsigned Opcode) {
- if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
- return true;
- switch (Opcode) {
- case Instruction::Freeze:
- case Instruction::GetElementPtr:
- case Instruction::ICmp:
- case Instruction::FCmp:
- case Instruction::Select:
- case VPInstruction::Not:
- case VPInstruction::Broadcast:
- case VPInstruction::MaskedCond:
- case VPInstruction::PtrAdd:
- return true;
- default:
- return false;
- }
-}
-
bool vputils::isSingleScalar(const VPValue *VPV) {
// Live-in, symbolic and region-values represent single-scalar values.
if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
return true;
- if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
+ // Use WideningInfo present directly on the recipe.
+ const VPRecipeBase *R = VPV->getDefiningRecipe();
+ if (R->isNarrow())
+ return true;
+
+ if (auto *Rep = dyn_cast<VPReplicateRecipe>(R)) {
const VPRegionBlock *RegionOfR = Rep->getRegion();
// Don't consider recipes in replicate regions as uniform yet; their first
// lane cannot be accessed when executing the replicate region for other
// lanes.
if (RegionOfR && RegionOfR->isReplicator())
return false;
- return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
- all_of(Rep->operands(), isSingleScalar));
- }
- if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
- return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
- if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
- return preservesUniformity(WidenR->getOpcode()) &&
- all_of(WidenR->operands(), isSingleScalar);
}
- if (auto *VPI = dyn_cast<VPInstruction>(VPV))
- return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- (preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isSingleScalar));
- if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
- return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
- return true;
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
- return Expr->isSingleScalar();
-
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ return R->maybeNarrow() && all_of(R->operands(), isSingleScalar);
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -419,19 +382,15 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
// Be conservative about side-effects, except for the
// known-side-effecting assumes and stores, which we know will be
// uniform.
- return R->isSingleScalar() &&
+ return R->isNarrow() &&
(!R->mayHaveSideEffects() ||
isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPWidenRecipe *R) {
- return preservesUniformity(R->getOpcode()) &&
+ .Case<VPWidenRecipe, VPInstruction>([](const auto *R) {
+ return R->maybeNarrow() &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
- .Case([](const VPInstruction *VPI) {
- return preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isUniformAcrossVFsAndUFs);
- })
.Case([](const VPWidenCastRecipe *R) {
// A cast is uniform according to its operand.
return isUniformAcrossVFsAndUFs(R->getOperand(0));
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index a1ddda7eda969..dd9c1332d54e2 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1697,7 +1697,8 @@ TEST_F(VPRecipeTest, CastVPReductionEVLRecipeToVPUser) {
} // namespace
struct VPDoubleValueDef : public VPRecipeBase {
- VPDoubleValueDef(ArrayRef<VPValue *> Operands) : VPRecipeBase(99, Operands) {
+ VPDoubleValueDef(ArrayRef<VPValue *> Operands)
+ : VPRecipeBase(99, VPRecipeBase::Wide, Operands) {
new VPRecipeValue(this);
new VPRecipeValue(this);
}
>From 58cacd953ea9ce34786920fc0b52ce505dc427b0 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sun, 3 May 2026 13:58:14 +0100
Subject: [PATCH 2/4] [VPlan] Introduce Agnostic bit
---
llvm/lib/Transforms/Vectorize/VPlan.h | 37 ++++++++++++++-----
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 15 ++++----
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 6 +--
5 files changed, 41 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 824dac1fd07f2..eecff9bd88150 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -409,8 +409,8 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Subclass identifier (for isa/dyn_cast).
const unsigned char SubclassID;
- /// Three bits of widening information, which takes values in VPWideningTy.
- unsigned char WideningInfo : 3;
+ /// Four bits of widening information, which takes values in VPWideningTy.
+ unsigned char WideningInfo : 4;
/// Each VPRecipe belongs to a single VPBasicBlock.
VPBasicBlock *Parent = nullptr;
@@ -470,22 +470,34 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
};
/// An enumeration for keeping track of the widening status of the recipe.
- /// The recipe necessarily produceds a scalar value if only the Narrow bit is
+ /// The recipe necessarily produces a scalar value if only the Narrow bit is
/// set, a wide value if only the Wide bit is set, and scalar values for each
/// unroll part if only the ReplicatePart bit is set. The Narrow bit can be
/// set on Wide and ReplicatePart recipes, which indicates that the recipe
- /// could be considered narrow if profitable.
+ /// could be considered narrow if profitable. Finally, there is a class of
+ /// instructions that necessarily take vector operands and produce a scalar
+ /// result, like (Insert|Extract)Element, or necessarily take a single scalar
+ /// operand and produce a vector, like Broadcast: there is no widening
+ /// decision to make on this class, and it is marked with the Agnostic bit.
using VPWideningTy = enum {
Narrow = 1 << 0,
Wide = 1 << 1,
ReplicatePart = 1 << 2,
+ Agnostic = 1 << 3
};
VPRecipeBase(const unsigned char SC, unsigned char WideInfo,
ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
: VPDef(), VPUser(Operands), SubclassID(SC), WideningInfo(WideInfo),
- DL(DL) {}
+ DL(DL) {
+ assert((!couldReplicatePerPart() || !isWide()) &&
+ "WideInfo cannot simultaneously be ReplicatePart and Wide");
+ assert((!couldReplicatePerPart() || !isAgnostic()) &&
+ "WideInfo cannot simultaenously be ReplicatePart and Agnostic");
+ assert((!isAgnostic() || (isNarrow() ^ isWide())) &&
+ "Agnostic WideInfo cannot be Narrow and Wide simultaenously");
+ }
~VPRecipeBase() override = default;
@@ -571,11 +583,18 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// Methods for querying and setting WideningInfo.
bool isNarrow() const { return WideningInfo == Narrow; }
+ bool isWide() const { return WideningInfo == Wide; }
+ bool isAgnostic() const { return WideningInfo & Agnostic; }
+ bool producesNarrowResult() const {
+ return !(WideningInfo & (Wide | ReplicatePart));
+ }
+ bool couldProduceNarrowResult() const { return WideningInfo & Narrow; }
+ bool couldReplicatePerPart() const { return WideningInfo & ReplicatePart; }
void markNarrow() { WideningInfo = Narrow; }
void markPossiblyNarrow() { WideningInfo |= Narrow; }
+ void markVectorToScalar() { WideningInfo = (Narrow | Agnostic); }
+ void markScalarToVector() { WideningInfo = (Wide | Agnostic); }
void markReplicatePart() { WideningInfo = ReplicatePart; }
- bool maybeNarrow() const { return WideningInfo & Narrow; }
- bool maybeReplicatePart() const { return WideningInfo & ReplicatePart; }
/// Set the recipe's debug location to \p NewDL.
void setDebugLoc(DebugLoc NewDL) { DL = NewDL; }
@@ -3460,8 +3479,8 @@ class VPExpressionRecipe : public VPSingleDefRecipe {
bool mayHaveSideEffects() const;
protected:
- /// Returns true if the result of this VPExpressionRecipe is a single-scalar.
- bool isSingleScalar() const;
+ /// Returns true if this recipe produces a scalar result.
+ bool isVectorToScalar() const;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 0dcc667808448..4f52fbd465243 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -430,7 +430,6 @@ static bool possiblyNarrowOpcode(unsigned Opcode) {
case Instruction::FCmp:
case Instruction::Select:
case VPInstruction::Not:
- case VPInstruction::Broadcast:
case VPInstruction::MaskedCond:
case VPInstruction::PtrAdd:
return true;
@@ -445,8 +444,12 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
: VPRecipeWithIRFlags(VPRecipeBase::VPInstructionSC, VPRecipeBase::Wide,
Operands, Flags, DL),
VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {
- if (isSingleScalar() || isVectorToScalar())
+ if (isSingleScalar())
markNarrow();
+ else if (isVectorToScalar())
+ markVectorToScalar();
+ else if (Opcode == VPInstruction::Broadcast)
+ markScalarToVector();
else if (is_contained({VPInstruction::Unpack, VPInstruction::PtrAdd}, Opcode))
markReplicatePart();
if (possiblyNarrowOpcode(Opcode))
@@ -3045,8 +3048,8 @@ VPExpressionRecipe::VPExpressionRecipe(
for (auto const &[LiveIn, Tmp] : zip(operands(), LiveInPlaceholders))
R->replaceUsesOfWith(LiveIn, Tmp);
- if (isSingleScalar())
- markNarrow();
+ if (isVectorToScalar())
+ markVectorToScalar();
}
void VPExpressionRecipe::decompose() {
@@ -3140,9 +3143,7 @@ bool VPExpressionRecipe::mayHaveSideEffects() const {
return false;
}
-bool VPExpressionRecipe::isSingleScalar() const {
- // Cannot use vputils::isSingleScalar(), because all external operands
- // of the expression will be live-ins while bundled.
+bool VPExpressionRecipe::isVectorToScalar() const {
auto *RR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
return RR && !RR->isPartialReduction();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c851878b3b975..88ef830d304f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4907,7 +4907,7 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
for (VPBasicBlock *VPBB :
concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion)) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!R.maybeReplicatePart())
+ if (!R.couldReplicatePerPart())
continue;
auto *DefR = cast<VPSingleDefRecipe>(&R);
auto UsesVectorOrInsideReplicateRegion = [DefR, LoopRegion](VPUser *U) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 8adeb3ef4d7c6..6014b2ffc4c17 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -862,7 +862,7 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
SmallVector<VPRecipeBase *> ToRemove;
for (VPBasicBlock *VPBB : VPBBsToUnroll) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (!R.maybeReplicatePart() ||
+ if (!R.couldReplicatePerPart() ||
(isa<VPInstruction>(&R) &&
!cast<VPInstruction>(&R)->doesGeneratePerAllLanes()))
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 8c73b667e960f..9f57608eccae9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -345,7 +345,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
// Use WideningInfo present directly on the recipe.
const VPRecipeBase *R = VPV->getDefiningRecipe();
- if (R->isNarrow())
+ if (R->producesNarrowResult() || R->isAgnostic())
return true;
if (auto *Rep = dyn_cast<VPReplicateRecipe>(R)) {
@@ -356,7 +356,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
if (RegionOfR && RegionOfR->isReplicator())
return false;
}
- return R->maybeNarrow() && all_of(R->operands(), isSingleScalar);
+ return R->couldProduceNarrowResult() && all_of(R->operands(), isSingleScalar);
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -388,7 +388,7 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
.Case<VPWidenRecipe, VPInstruction>([](const auto *R) {
- return R->maybeNarrow() &&
+ return (R->producesNarrowResult() || R->isAgnostic()) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
})
.Case([](const VPWidenCastRecipe *R) {
>From 38476b77df62d3b7a962de1a4ac81e95329eeb7a Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sun, 3 May 2026 16:26:25 +0100
Subject: [PATCH 3/4] [VPlan] Move asserts, fix prose to include stores
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 8 ++++++++
llvm/lib/Transforms/Vectorize/VPlan.h | 22 +++++++++-------------
2 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 91fa4c57e5bf9..2573c60e9e1a3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -557,6 +557,14 @@ void VPBasicBlock::executeRecipes(VPTransformState *State, BasicBlock *BB) {
for (VPRecipeBase &Recipe : Recipes) {
State->setDebugLocFrom(Recipe.getDebugLoc());
+ assert(Recipe.WideningInfo && "WideInfo needs to be set");
+ assert(
+ (!Recipe.couldReplicatePerPart() ||
+ (!Recipe.isWide() && !Recipe.isAgnostic())) &&
+ "WideInfo cannot simultaneously be ReplicatePart and Wide or Agnostic");
+ assert((!Recipe.isAgnostic() || (Recipe.couldProduceNarrowResult() ^
+ Recipe.couldProducWideResult())) &&
+ "Agnostic WideInfo cannot be Narrow and Wide simultaenously");
Recipe.execute(*State);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index eecff9bd88150..da6b2005e2cbd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -474,11 +474,13 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
/// set, a wide value if only the Wide bit is set, and scalar values for each
/// unroll part if only the ReplicatePart bit is set. The Narrow bit can be
/// set on Wide and ReplicatePart recipes, which indicates that the recipe
- /// could be considered narrow if profitable. Finally, there is a class of
- /// instructions that necessarily take vector operands and produce a scalar
- /// result, like (Insert|Extract)Element, or necessarily take a single scalar
- /// operand and produce a vector, like Broadcast: there is no widening
- /// decision to make on this class, and it is marked with the Agnostic bit.
+ /// could be considered narrow if profitable. For instructions not producing
+ /// values, like an assume or store, the bits talk about the inherent widening
+ /// status of the recipe. Finally, there is a class of instructions that
+ /// necessarily take vector operands and produce a scalar result, like
+ /// (Insert|Extract)Element, or necessarily take a single scalar operand and
+ /// produce a vector, like Broadcast: there is no widening decision to make on
+ /// this class, and it is marked with the Agnostic bit.
using VPWideningTy = enum {
Narrow = 1 << 0,
Wide = 1 << 1,
@@ -490,14 +492,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
: VPDef(), VPUser(Operands), SubclassID(SC), WideningInfo(WideInfo),
- DL(DL) {
- assert((!couldReplicatePerPart() || !isWide()) &&
- "WideInfo cannot simultaneously be ReplicatePart and Wide");
- assert((!couldReplicatePerPart() || !isAgnostic()) &&
- "WideInfo cannot simultaenously be ReplicatePart and Agnostic");
- assert((!isAgnostic() || (isNarrow() ^ isWide())) &&
- "Agnostic WideInfo cannot be Narrow and Wide simultaenously");
- }
+ DL(DL) {}
~VPRecipeBase() override = default;
@@ -589,6 +584,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
return !(WideningInfo & (Wide | ReplicatePart));
}
bool couldProduceNarrowResult() const { return WideningInfo & Narrow; }
+ bool couldProducWideResult() const { return WideningInfo & Wide; }
bool couldReplicatePerPart() const { return WideningInfo & ReplicatePart; }
void markNarrow() { WideningInfo = Narrow; }
void markPossiblyNarrow() { WideningInfo |= Narrow; }
>From f1501c7fb0ae59b9f94bf6893dea35a3bf985b22 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sun, 3 May 2026 16:57:18 +0100
Subject: [PATCH 4/4] [VPlan] Strip isNarrow, isWide to reduce confusion
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 6 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 24 +++++++-------
.../Transforms/Vectorize/VPlanTransforms.cpp | 33 ++++++++++---------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +-
5 files changed, 34 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2573c60e9e1a3..df7cdb59fcf02 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -560,7 +560,7 @@ void VPBasicBlock::executeRecipes(VPTransformState *State, BasicBlock *BB) {
assert(Recipe.WideningInfo && "WideInfo needs to be set");
assert(
(!Recipe.couldReplicatePerPart() ||
- (!Recipe.isWide() && !Recipe.isAgnostic())) &&
+ (!Recipe.couldProducWideResult() && !Recipe.isAgnostic())) &&
"WideInfo cannot simultaneously be ReplicatePart and Wide or Agnostic");
assert((!Recipe.isAgnostic() || (Recipe.couldProduceNarrowResult() ^
Recipe.couldProducWideResult())) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index da6b2005e2cbd..f85689a2eba4a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -577,8 +577,6 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
DebugLoc getDebugLoc() const { return DL; }
/// Methods for querying and setting WideningInfo.
- bool isNarrow() const { return WideningInfo == Narrow; }
- bool isWide() const { return WideningInfo == Wide; }
bool isAgnostic() const { return WideningInfo & Agnostic; }
bool producesNarrowResult() const {
return !(WideningInfo & (Wide | ReplicatePart));
@@ -3247,7 +3245,7 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
VPReplicateRecipe *clone() override {
auto *Copy = new VPReplicateRecipe(
- getUnderlyingInstr(), operands(), isNarrow(),
+ getUnderlyingInstr(), operands(), producesNarrowResult(),
isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc());
Copy->transferFlags(*this);
return Copy;
@@ -3270,7 +3268,7 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
bool usesFirstLaneOnly(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
"Op must be an operand of the recipe");
- return isNarrow();
+ return producesNarrowResult();
}
/// Returns true if the recipe uses scalars of operand \p Op.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4f52fbd465243..dc8095d64ac17 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3367,13 +3367,14 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
Instruction *UI = getUnderlyingInstr();
if (!State.Lane) {
- assert(isNarrow() && "VPReplicateRecipes outside replicate regions "
- "must have already been unrolled");
+ assert(producesNarrowResult() &&
+ "VPReplicateRecipes outside replicate regions "
+ "must have already been unrolled");
scalarizeInstruction(UI, this, VPLane(0), State);
return;
}
- assert((State.VF.isScalar() || !isNarrow()) &&
+ assert((State.VF.isScalar() || !producesNarrowResult()) &&
"uniform recipe shouldn't be predicated");
assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
scalarizeInstruction(UI, this, *State.Lane, State);
@@ -3484,7 +3485,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
// transform, avoid computing their cost multiple times for now.
Ctx.SkipCostComputation.insert(UI);
- if (VF.isScalable() && !isNarrow())
+ if (VF.isScalable() && !producesNarrowResult())
return InstructionCost::getInvalid();
switch (UI->getOpcode()) {
@@ -3530,7 +3531,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
Type *ResultTy = Ctx.Types.inferScalarType(this);
InstructionCost ScalarCallCost =
Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
- if (isNarrow()) {
+ if (producesNarrowResult()) {
if (CalledFn->isIntrinsic())
ScalarCallCost = std::min(
ScalarCallCost,
@@ -3560,14 +3561,14 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
case Instruction::FCmp:
return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
Ctx) *
- (isNarrow() ? 1 : VF.getFixedValue());
+ (producesNarrowResult() ? 1 : VF.getFixedValue());
case Instruction::SDiv:
case Instruction::UDiv:
case Instruction::SRem:
case Instruction::URem: {
InstructionCost ScalarCost =
getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1), Ctx);
- if (isNarrow())
+ if (producesNarrowResult())
return ScalarCost;
// If any of the operands is from a different replicate region and has its
@@ -3642,13 +3643,14 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
return UniformCost;
}
- Type *PtrTy = isNarrow() ? ScalarPtrTy : toVectorTy(ScalarPtrTy, VF);
+ Type *PtrTy =
+ producesNarrowResult() ? ScalarPtrTy : toVectorTy(ScalarPtrTy, VF);
InstructionCost ScalarCost =
ScalarMemOpCost +
Ctx.TTI.getAddressComputationCost(
PtrTy, UsedByLoadStoreAddress ? nullptr : Ctx.PSE.getSE(), PtrSCEV,
Ctx.CostKind);
- if (isNarrow())
+ if (producesNarrowResult())
return ScalarCost;
SmallVector<const VPValue *> OpsToScalarize;
@@ -3711,7 +3713,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
case Instruction::AddrSpaceCast: {
return getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
Ctx) *
- (isNarrow() ? 1 : VF.getFixedValue());
+ (producesNarrowResult() ? 1 : VF.getFixedValue());
}
case Instruction::ExtractValue:
case Instruction::InsertValue:
@@ -3724,7 +3726,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPReplicateRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
- O << Indent << (isNarrow() ? "CLONE " : "REPLICATE ");
+ O << Indent << (producesNarrowResult() ? "CLONE " : "REPLICATE ");
if (!getUnderlyingInstr()->getType()->isVoidTy()) {
printAsOperand(O, SlotTracker);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 88ef830d304f0..fdf137b754c03 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -310,7 +310,7 @@ static bool sinkScalarOperands(VPlan &Plan) {
return;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(Candidate))
- if (!ScalarVFOnly && RepR->isNarrow())
+ if (!ScalarVFOnly && RepR->producesNarrowResult())
return;
WorkList.insert({SinkTo, Candidate});
@@ -520,8 +520,8 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
// mask but in the replicate region.
auto *RecipeWithoutMask = new VPReplicateRecipe(
PredRecipe->getUnderlyingInstr(), drop_end(PredRecipe->operands()),
- PredRecipe->isNarrow(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
- PredRecipe->getDebugLoc());
+ PredRecipe->producesNarrowResult(), nullptr /*Mask*/, *PredRecipe,
+ *PredRecipe, PredRecipe->getDebugLoc());
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
auto *Exiting = Plan.createVPBasicBlock(Twine(RegionName) + ".continue");
@@ -908,7 +908,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
// Skip recipes that shouldn't be narrowed.
if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
Def->getNumUsers() == 0 || !Def->getUnderlyingValue() ||
- (RepR && (RepR->isNarrow() || RepR->isPredicated())))
+ (RepR && (RepR->producesNarrowResult() || RepR->isPredicated())))
continue;
// Skip recipes that may have other lanes than their first used.
@@ -1914,7 +1914,7 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
auto *IRV = dyn_cast<VPIRValue>(Op);
bool LiveInNeedsBroadcast = IRV && !isa<Constant>(IRV->getValue());
auto *OpR = dyn_cast<VPReplicateRecipe>(Op);
- return LiveInNeedsBroadcast || (OpR && OpR->isNarrow());
+ return LiveInNeedsBroadcast || (OpR && OpR->producesNarrowResult());
}))
continue;
@@ -2526,7 +2526,7 @@ static void licm(VPlan &Plan) {
// replicates to single-scalar replicates.
// TODO: When unrolling, replicateByVF doesn't handle sunk
// non-single-scalar replicates correctly.
- if (!RepR->isNarrow())
+ if (!RepR->producesNarrowResult())
continue;
}
@@ -4605,7 +4605,7 @@ void VPlanTransforms::hoistInvariantLoads(VPlan &Plan) {
for (VPRecipeBase &R : *VPBB) {
// Only handle single-scalar replicated loads with invariant addresses.
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
- if (RepR->isPredicated() || !RepR->isNarrow() ||
+ if (RepR->isPredicated() || !RepR->producesNarrowResult() ||
RepR->getOpcode() != Instruction::Load)
continue;
@@ -4747,10 +4747,10 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
// Find the load with minimum alignment to use.
auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
- bool IsNarrow = EarliestLoad->isNarrow();
+ bool producesNarrowResult = EarliestLoad->producesNarrowResult();
assert(all_of(Group,
- [IsNarrow](VPReplicateRecipe *R) {
- return R->isNarrow() == IsNarrow;
+ [producesNarrowResult](VPReplicateRecipe *R) {
+ return R->producesNarrowResult() == producesNarrowResult;
}) &&
"all members in group must agree on narrowing");
@@ -4758,7 +4758,7 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
// metadata.
auto *UnpredicatedLoad = new VPReplicateRecipe(
LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(0)},
- IsNarrow, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
+ producesNarrowResult, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
UnpredicatedLoad->insertBefore(EarliestLoad);
@@ -4815,9 +4815,9 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
VPValue *SelectedValue = Group[0]->getOperand(0);
VPBuilder Builder(InsertBB, LastStore->getIterator());
- bool IsNarrow = Group[0]->isNarrow();
+ bool producesNarrowResult = Group[0]->producesNarrowResult();
for (unsigned I = 1; I < Group.size(); ++I) {
- assert(IsNarrow == Group[I]->isNarrow() &&
+ assert(producesNarrowResult == Group[I]->producesNarrowResult() &&
"all members in group must agree on narrowing");
VPValue *Mask = Group[I]->getMask();
VPValue *Value = Group[I]->getOperand(0);
@@ -4831,7 +4831,7 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
// Create unconditional store with selected value and common metadata.
auto *UnpredicatedStore = new VPReplicateRecipe(
StoreWithMinAlign->getUnderlyingInstr(),
- {SelectedValue, LastStore->getOperand(1)}, IsNarrow,
+ {SelectedValue, LastStore->getOperand(1)}, producesNarrowResult,
/*Mask=*/nullptr, *LastStore, CommonMetadata);
UnpredicatedStore->insertBefore(*InsertBB, LastStore->getIterator());
@@ -5239,7 +5239,7 @@ static bool isAlreadyNarrow(VPValue *VPV) {
if (isa<VPIRValue>(VPV))
return true;
auto *RepR = dyn_cast<VPReplicateRecipe>(VPV);
- return RepR && RepR->isNarrow();
+ return RepR && RepR->producesNarrowResult();
}
// Convert a wide recipe defining a VPValue \p V feeding an interleave group to
@@ -5275,7 +5275,8 @@ narrowInterleaveGroupOp(VPValue *V, SmallPtrSetImpl<VPValue *> &NarrowedOps) {
}
if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
- assert(RepR->isNarrow() && RepR->getOpcode() == Instruction::Load &&
+ assert(RepR->producesNarrowResult() &&
+ RepR->getOpcode() == Instruction::Load &&
"must be a single scalar load");
NarrowedOps.insert(RepR);
return RepR;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 9f57608eccae9..cd67dd591b147 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -382,7 +382,7 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
// Be conservative about side-effects, except for the
// known-side-effecting assumes and stores, which we know will be
// uniform.
- return R->isNarrow() &&
+ return R->producesNarrowResult() &&
(!R->mayHaveSideEffects() ||
isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
More information about the llvm-commits
mailing list