[llvm] c0506a1 - [VPlan] Separate out logic to manage IR flags to VPIRFlags (NFC). (#140621)
via llvm-commits
llvm-commits at lists.llvm.org
Sun May 25 03:13:14 PDT 2025
Author: Florian Hahn
Date: 2025-05-25T11:13:11+01:00
New Revision: c0506a11f4e9e7a3b3d8a6697f12963f509e58d3
URL: https://github.com/llvm/llvm-project/commit/c0506a11f4e9e7a3b3d8a6697f12963f509e58d3
DIFF: https://github.com/llvm/llvm-project/commit/c0506a11f4e9e7a3b3d8a6697f12963f509e58d3.diff
LOG: [VPlan] Separate out logic to manage IR flags to VPIRFlags (NFC). (#140621)
This patch moves the logic to manage IR flags to a separate VPIRFlags
class. For now, VPRecipeWithIRFlags is the only class that inherits
VPIRFlags. The new class allows for simpler passing of flags when
constructing recipes, simplifying the constructors for various recipes
(VPInstruction in particular, which now just has 2 constructors, one
taking an extra VPIRFlags argument.
This mirrors the approach taken for VPIRMetadata and makes it easier to
extend in the future. The patch also adds a unified flagsValidForOpcode
to check if the flags in a VPIRFlags match the provided opcode.
PR: https://github.com/llvm/llvm-project/pull/140621
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bae53c600c18c..c751f053cb65a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -164,25 +164,19 @@ class VPBuilder {
DebugLoc DL, const Twine &Name = "") {
return createInstruction(Opcode, Operands, DL, Name);
}
- VPInstruction *createNaryOp(unsigned Opcode,
- std::initializer_list<VPValue *> Operands,
- std::optional<FastMathFlags> FMFs = {},
- DebugLoc DL = {}, const Twine &Name = "") {
- if (FMFs)
- return tryInsertInstruction(
- new VPInstruction(Opcode, Operands, *FMFs, DL, Name));
- return createInstruction(Opcode, Operands, DL, Name);
+ VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags, DebugLoc DL = {},
+ const Twine &Name = "") {
+ return tryInsertInstruction(
+ new VPInstruction(Opcode, Operands, Flags, DL, Name));
}
+
VPInstruction *createNaryOp(unsigned Opcode,
std::initializer_list<VPValue *> Operands,
- Type *ResultTy,
- std::optional<FastMathFlags> FMFs = {},
+ Type *ResultTy, const VPIRFlags &Flags = {},
DebugLoc DL = {}, const Twine &Name = "") {
- if (FMFs)
- return tryInsertInstruction(new VPInstructionWithType(
- Opcode, Operands, ResultTy, *FMFs, DL, Name));
return tryInsertInstruction(
- new VPInstructionWithType(Opcode, Operands, ResultTy, DL, Name));
+ new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name));
}
VPInstruction *createOverflowingOp(unsigned Opcode,
@@ -236,18 +230,20 @@ class VPBuilder {
assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
return tryInsertInstruction(
- new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name));
+ new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name));
}
VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {},
const Twine &Name = "") {
return tryInsertInstruction(
- new VPInstruction(Ptr, Offset, GEPNoWrapFlags::none(), DL, Name));
+ new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
+ GEPNoWrapFlags::none(), DL, Name));
}
VPValue *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {},
const Twine &Name = "") {
return tryInsertInstruction(
- new VPInstruction(Ptr, Offset, GEPNoWrapFlags::inBounds(), DL, Name));
+ new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
+ GEPNoWrapFlags::inBounds(), DL, Name));
}
VPInstruction *createScalarPhi(ArrayRef<VPValue *> IncomingValues,
@@ -269,7 +265,7 @@ class VPBuilder {
VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
Type *ResultTy, DebugLoc DL) {
return tryInsertInstruction(
- new VPInstructionWithType(Opcode, Op, ResultTy, DL));
+ new VPInstructionWithType(Opcode, Op, ResultTy, {}, DL));
}
VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c4e66cd89e69c..28fad085b4b7b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -577,8 +577,8 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
#endif
};
-/// Class to record LLVM IR flag for a recipe along with it.
-class VPRecipeWithIRFlags : public VPSingleDefRecipe {
+/// Class to record and manage LLVM IR flags.
+class VPIRFlags {
enum class OperationType : unsigned char {
Cmp,
OverflowingBinOp,
@@ -637,23 +637,10 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
unsigned AllFlags;
};
-protected:
- void transferFlags(VPRecipeWithIRFlags &Other) {
- OpType = Other.OpType;
- AllFlags = Other.AllFlags;
- }
-
public:
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- DebugLoc DL = {})
- : VPSingleDefRecipe(SC, Operands, DL) {
- OpType = OperationType::Other;
- AllFlags = 0;
- }
+ VPIRFlags() : OpType(OperationType::Other), AllFlags(0) {}
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- Instruction &I)
- : VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()) {
+ VPIRFlags(Instruction &I) {
if (auto *Op = dyn_cast<CmpInst>(&I)) {
OpType = OperationType::Cmp;
CmpPredicate = Op->getPredicate();
@@ -681,63 +668,27 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
}
}
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- CmpInst::Predicate Pred, DebugLoc DL = {})
- : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::Cmp),
- CmpPredicate(Pred) {}
+ VPIRFlags(CmpInst::Predicate Pred)
+ : OpType(OperationType::Cmp), CmpPredicate(Pred) {}
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- WrapFlagsTy WrapFlags, DebugLoc DL = {})
- : VPSingleDefRecipe(SC, Operands, DL),
- OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {}
+ VPIRFlags(WrapFlagsTy WrapFlags)
+ : OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {}
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL = {})
- : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::FPMathOp),
- FMFs(FMFs) {}
+ VPIRFlags(FastMathFlags FMFs) : OpType(OperationType::FPMathOp), FMFs(FMFs) {}
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- DisjointFlagsTy DisjointFlags, DebugLoc DL = {})
- : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::DisjointOp),
- DisjointFlags(DisjointFlags) {}
+ VPIRFlags(DisjointFlagsTy DisjointFlags)
+ : OpType(OperationType::DisjointOp), DisjointFlags(DisjointFlags) {}
- template <typename IterT>
- VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
- NonNegFlagsTy NonNegFlags, DebugLoc DL = {})
- : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::NonNegOp),
- NonNegFlags(NonNegFlags) {}
+ VPIRFlags(NonNegFlagsTy NonNegFlags)
+ : OpType(OperationType::NonNegOp), NonNegFlags(NonNegFlags) {}
-protected:
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- GEPNoWrapFlags GEPFlags, DebugLoc DL = {})
- : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::GEPOp),
- GEPFlags(GEPFlags) {}
+ VPIRFlags(GEPNoWrapFlags GEPFlags)
+ : OpType(OperationType::GEPOp), GEPFlags(GEPFlags) {}
public:
- static inline bool classof(const VPRecipeBase *R) {
- return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
- R->getVPDefID() == VPRecipeBase::VPReductionSC ||
- R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
- R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
- R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC ||
- R->getVPDefID() == VPRecipeBase::VPVectorPointerSC ||
- R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC ||
- R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC;
- }
-
- static inline bool classof(const VPUser *U) {
- auto *R = dyn_cast<VPRecipeBase>(U);
- return R && classof(R);
- }
-
- static inline bool classof(const VPValue *V) {
- auto *R = dyn_cast_or_null<VPRecipeBase>(V->getDefiningRecipe());
- return R && classof(R);
+ void transferFlags(VPIRFlags &Other) {
+ OpType = Other.OpType;
+ AllFlags = Other.AllFlags;
}
/// Drop all poison-generating flags.
@@ -851,11 +802,60 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
return DisjointFlags.IsDisjoint;
}
+#if !defined(NDEBUG)
+ /// Returns true if the set flags are valid for \p Opcode.
+ bool flagsValidForOpcode(unsigned Opcode) const;
+#endif
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void printFlags(raw_ostream &O) const;
#endif
};
+/// A pure-virtual common base class for recipes defining a single VPValue and
+/// using IR flags.
+struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
+ VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ DebugLoc DL = {})
+ : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags() {}
+
+ VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ Instruction &I)
+ : VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()), VPIRFlags(I) {}
+
+ VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags, DebugLoc DL = {})
+ : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
+
+ static inline bool classof(const VPRecipeBase *R) {
+ return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
+ R->getVPDefID() == VPRecipeBase::VPWidenSC ||
+ R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
+ R->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
+ R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
+ R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
+ R->getVPDefID() == VPRecipeBase::VPReductionSC ||
+ R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
+ R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
+ R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC ||
+ R->getVPDefID() == VPRecipeBase::VPVectorPointerSC ||
+ R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC ||
+ R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC;
+ }
+
+ static inline bool classof(const VPUser *U) {
+ auto *R = dyn_cast<VPRecipeBase>(U);
+ return R && classof(R);
+ }
+
+ static inline bool classof(const VPValue *V) {
+ auto *R = dyn_cast_or_null<VPRecipeBase>(V->getDefiningRecipe());
+ return R && classof(R);
+ }
+
+ void execute(VPTransformState &State) override = 0;
+};
+
/// Helper to access the operand that contains the unroll part for this recipe
/// after unrolling.
template <unsigned PartOpIdx> class VPUnrollPartAccessor {
@@ -958,54 +958,21 @@ class VPInstruction : public VPRecipeWithIRFlags,
/// value for lane \p Lane.
Value *generatePerLane(VPTransformState &State, const VPLane &Lane);
-#if !defined(NDEBUG)
- /// Return true if the VPInstruction is a floating point math operation, i.e.
- /// has fast-math flags.
- bool isFPMathOp() const;
-#endif
-
public:
- VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL = {},
const Twine &Name = "")
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
Opcode(Opcode), Name(Name.str()) {}
- VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- DebugLoc DL = {}, const Twine &Name = "")
- : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}
-
- VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
- VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
-
- VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "")
- : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL),
- Opcode(Opcode), Name(Name.str()) {}
-
- VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- DisjointFlagsTy DisjointFlag, DebugLoc DL = {},
- const Twine &Name = "")
- : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DisjointFlag, DL),
- Opcode(Opcode), Name(Name.str()) {
- assert(Opcode == Instruction::Or && "only OR opcodes can be disjoint");
- }
-
- VPInstruction(VPValue *Ptr, VPValue *Offset, GEPNoWrapFlags Flags,
- DebugLoc DL = {}, const Twine &Name = "")
- : VPRecipeWithIRFlags(VPDef::VPInstructionSC,
- ArrayRef<VPValue *>({Ptr, Offset}), Flags, DL),
- Opcode(VPInstruction::PtrAdd), Name(Name.str()) {}
-
- VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags, DebugLoc DL = {},
+ const Twine &Name = "");
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
VPInstruction *clone() override {
SmallVector<VPValue *, 2> Operands(operands());
- auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name);
- New->transferFlags(*this);
- return New;
+ return new VPInstruction(Opcode, Operands, *this, getDebugLoc(), Name);
}
unsigned getOpcode() const { return Opcode; }
@@ -1082,13 +1049,9 @@ class VPInstructionWithType : public VPInstruction {
public:
VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
- Type *ResultTy, DebugLoc DL, const Twine &Name = "")
- : VPInstruction(Opcode, Operands, DL, Name), ResultTy(ResultTy) {}
- VPInstructionWithType(unsigned Opcode,
- std::initializer_list<VPValue *> Operands,
- Type *ResultTy, FastMathFlags FMFs, DebugLoc DL = {},
+ Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL,
const Twine &Name = "")
- : VPInstruction(Opcode, Operands, FMFs, DL, Name), ResultTy(ResultTy) {}
+ : VPInstruction(Opcode, Operands, Flags, DL, Name), ResultTy(ResultTy) {}
static inline bool classof(const VPRecipeBase *R) {
// VPInstructionWithType are VPInstructions with specific opcodes requiring
@@ -1113,8 +1076,9 @@ class VPInstructionWithType : public VPInstruction {
VPInstruction *clone() override {
SmallVector<VPValue *, 2> Operands(operands());
- auto *New = new VPInstructionWithType(
- getOpcode(), Operands, getResultType(), getDebugLoc(), getName());
+ auto *New =
+ new VPInstructionWithType(getOpcode(), Operands, getResultType(), *this,
+ getDebugLoc(), getName());
New->setUnderlyingValue(getUnderlyingValue());
return New;
}
@@ -1373,15 +1337,12 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, DL), VPIRMetadata(),
- Opcode(Opcode), ResultTy(ResultTy) {}
-
- VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- bool IsNonNeg, DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, NonNegFlagsTy(IsNonNeg),
- DL),
- Opcode(Opcode), ResultTy(ResultTy) {}
+ const VPIRFlags &Flags = {}, DebugLoc DL = {})
+ : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL),
+ VPIRMetadata(), Opcode(Opcode), ResultTy(ResultTy) {
+ assert(flagsValidForOpcode(Opcode) &&
+ "Set flags not supported for the provided opcode");
+ }
~VPWidenCastRecipe() override = default;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 5c2ddb62c7155..26176e7859d74 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -368,7 +368,7 @@ void VPPartialReductionRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
-FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const {
+FastMathFlags VPIRFlags::getFastMathFlags() const {
assert(OpType == OperationType::FPMathOp &&
"recipe doesn't have fast math flags");
FastMathFlags Res;
@@ -406,23 +406,13 @@ template class VPUnrollPartAccessor<2>;
template class VPUnrollPartAccessor<3>;
}
-VPInstruction::VPInstruction(unsigned Opcode, CmpInst::Predicate Pred,
- VPValue *A, VPValue *B, DebugLoc DL,
+VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags, DebugLoc DL,
const Twine &Name)
- : VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef<VPValue *>({A, B}),
- Pred, DL),
+ : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, Flags, DL),
Opcode(Opcode), Name(Name.str()) {
- assert(Opcode == Instruction::ICmp &&
- "only ICmp predicates supported at the moment");
-}
-
-VPInstruction::VPInstruction(unsigned Opcode,
- std::initializer_list<VPValue *> Operands,
- FastMathFlags FMFs, DebugLoc DL, const Twine &Name)
- : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL),
- Opcode(Opcode), Name(Name.str()) {
- // Make sure the VPInstruction is a floating-point operation.
- assert(isFPMathOp() && "this op can't take fast-math flags");
+ assert(flagsValidForOpcode(getOpcode()) &&
+ "Set flags not supported for the provided opcode");
}
bool VPInstruction::doesGeneratePerAllLanes() const {
@@ -855,24 +845,11 @@ bool VPInstruction::isSingleScalar() const {
getOpcode() == Instruction::PHI;
}
-#if !defined(NDEBUG)
-bool VPInstruction::isFPMathOp() const {
- // Inspired by FPMathOperator::classof. Notable
diff erences are that we don't
- // support Call, PHI and Select opcodes here yet.
- return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
- Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
- Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
- Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
- Opcode == VPInstruction::WideIVStep;
-}
-#endif
-
void VPInstruction::execute(VPTransformState &State) {
assert(!State.Lane && "VPInstruction executing an Lane");
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
- assert((hasFastMathFlags() == isFPMathOp() ||
- getOpcode() == Instruction::Select) &&
- "Recipe not a FPMathOp but has fast-math flags?");
+ assert(flagsValidForOpcode(getOpcode()) &&
+ "Set flags not supported for the provided opcode");
if (hasFastMathFlags())
State.Builder.setFastMathFlags(getFastMathFlags());
bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
@@ -1593,8 +1570,7 @@ InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF,
{TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI);
}
-VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
- const FastMathFlags &FMF) {
+VPIRFlags::FastMathFlagsTy::FastMathFlagsTy(const FastMathFlags &FMF) {
AllowReassoc = FMF.allowReassoc();
NoNaNs = FMF.noNaNs();
NoInfs = FMF.noInfs();
@@ -1604,8 +1580,39 @@ VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
ApproxFunc = FMF.approxFunc();
}
+#if !defined(NDEBUG)
+bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
+ switch (OpType) {
+ case OperationType::OverflowingBinOp:
+ return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
+ Opcode == Instruction::Mul ||
+ Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;
+ case OperationType::DisjointOp:
+ return Opcode == Instruction::Or;
+ case OperationType::PossiblyExactOp:
+ return Opcode == Instruction::AShr;
+ case OperationType::GEPOp:
+ return Opcode == Instruction::GetElementPtr ||
+ Opcode == VPInstruction::PtrAdd;
+ case OperationType::FPMathOp:
+ return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
+ Opcode == Instruction::FSub || Opcode == Instruction::FNeg ||
+ Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
+ Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
+ Opcode == VPInstruction::WideIVStep;
+ case OperationType::NonNegOp:
+ return Opcode == Instruction::ZExt;
+ break;
+ case OperationType::Cmp:
+ return Opcode == Instruction::ICmp;
+ case OperationType::Other:
+ return true;
+ }
+}
+#endif
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPRecipeWithIRFlags::printFlags(raw_ostream &O) const {
+void VPIRFlags::printFlags(raw_ostream &O) const {
switch (OpType) {
case OperationType::Cmp:
O << " " << CmpInst::getPredicateName(getPredicate());
@@ -1642,8 +1649,7 @@ void VPRecipeWithIRFlags::printFlags(raw_ostream &O) const {
case OperationType::Other:
break;
}
- if (getNumOperands() > 0)
- O << " ";
+ O << " ";
}
#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 52d61d96c8083..d3f15eafb0214 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1343,8 +1343,9 @@ static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
assert(OldStepVector->getNumUsers() == 1 &&
"step vector should only be used by single "
"VPWidenIntOrFpInductionRecipe");
- auto *NewStepVector = new VPInstructionWithType(
- VPInstruction::StepVector, {}, NewIVTy, OldStepVector->getDebugLoc());
+ auto *NewStepVector =
+ new VPInstructionWithType(VPInstruction::StepVector, {}, NewIVTy, {},
+ OldStepVector->getDebugLoc());
NewStepVector->insertAfter(OldStepVector->getDefiningRecipe());
OldStepVector->replaceAllUsesWith(NewStepVector);
OldStepVector->eraseFromParent();
@@ -2524,11 +2525,12 @@ static void expandVPExtendedReduction(VPExtendedReductionRecipe *ExtRed) {
// Only ZExt contains non-neg flags.
if (ExtRed->isZExt())
Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(),
- ExtRed->getResultType(), ExtRed->isNonNeg(),
+ ExtRed->getResultType(), *ExtRed,
ExtRed->getDebugLoc());
else
Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(),
- ExtRed->getResultType(), ExtRed->getDebugLoc());
+ ExtRed->getResultType(), {},
+ ExtRed->getDebugLoc());
auto *Red = new VPReductionRecipe(
ExtRed->getRecurrenceKind(), FastMathFlags(), ExtRed->getChainOp(), Ext,
@@ -2551,12 +2553,12 @@ expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) {
if (MulAcc->isExtended()) {
Type *RedTy = MulAcc->getResultType();
if (MulAcc->isZExt())
- Op0 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp0(),
- RedTy, MulAcc->isNonNeg(),
- MulAcc->getDebugLoc());
+ Op0 = new VPWidenCastRecipe(
+ MulAcc->getExtOpcode(), MulAcc->getVecOp0(), RedTy,
+ VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()), MulAcc->getDebugLoc());
else
Op0 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp0(),
- RedTy, MulAcc->getDebugLoc());
+ RedTy, {}, MulAcc->getDebugLoc());
Op0->getDefiningRecipe()->insertBefore(MulAcc);
// Prevent reduce.add(mul(ext(A), ext(A))) generate duplicate
// VPWidenCastRecipe.
@@ -2564,12 +2566,13 @@ expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) {
Op1 = Op0;
} else {
if (MulAcc->isZExt())
- Op1 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp1(),
- RedTy, MulAcc->isNonNeg(),
- MulAcc->getDebugLoc());
+ Op1 = new VPWidenCastRecipe(
+ MulAcc->getExtOpcode(), MulAcc->getVecOp1(), RedTy,
+ VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()),
+ MulAcc->getDebugLoc());
else
Op1 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp1(),
- RedTy, MulAcc->getDebugLoc());
+ RedTy, {}, MulAcc->getDebugLoc());
Op1->getDefiningRecipe()->insertBefore(MulAcc);
}
} else {
@@ -2643,14 +2646,14 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
Builder.createWidenCast(Instruction::Trunc, ScalarStep, IVTy);
}
- std::optional<FastMathFlags> FMFs;
+ VPIRFlags Flags;
if (IVTy->isFloatingPointTy())
- FMFs = VPI->getFastMathFlags();
+ Flags = {VPI->getFastMathFlags()};
unsigned MulOpc =
IVTy->isFloatingPointTy() ? Instruction::FMul : Instruction::Mul;
VPInstruction *Mul = Builder.createNaryOp(
- MulOpc, {VectorStep, ScalarStep}, FMFs, R.getDebugLoc());
+ MulOpc, {VectorStep, ScalarStep}, Flags, R.getDebugLoc());
VectorStep = Mul;
VPI->replaceAllUsesWith(VectorStep);
ToRemove.push_back(VPI);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index ce83c276297c0..e1fb3d476c58d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -151,14 +151,14 @@ void UnrollState::unrollWidenInductionByUF(
IV->getParent()->getEnclosingLoopRegion()->getSinglePredecessor());
Type *IVTy = TypeInfo.inferScalarType(IV);
auto &ID = IV->getInductionDescriptor();
- std::optional<FastMathFlags> FMFs;
+ VPIRFlags Flags;
if (isa_and_present<FPMathOperator>(ID.getInductionBinOp()))
- FMFs = ID.getInductionBinOp()->getFastMathFlags();
+ Flags = ID.getInductionBinOp()->getFastMathFlags();
VPValue *ScalarStep = IV->getStepValue();
VPBuilder Builder(PH);
VPInstruction *VectorStep = Builder.createNaryOp(
- VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, IVTy, FMFs,
+ VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, IVTy, Flags,
IV->getDebugLoc());
ToSkip.insert(VectorStep);
@@ -188,7 +188,7 @@ void UnrollState::unrollWidenInductionByUF(
Prev,
VectorStep,
},
- FMFs, IV->getDebugLoc(), Name);
+ Flags, IV->getDebugLoc(), Name);
ToSkip.insert(Add);
addRecipeForPart(IV, Add, Part);
Prev = Add;
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 7218311518594..8b4fd066aaa26 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -752,13 +752,13 @@ edge [fontname=Courier, fontsize=30]
compound=true
N0 [label =
"preheader:\l" +
- " EMIT vp\<%1\> = add\l" +
+ " EMIT vp\<%1\> = add \l" +
"Successor(s): bb1\l"
]
N0 -> N1 [ label=""]
N1 [label =
"bb1:\l" +
- " EMIT vp\<%2\> = add\l" +
+ " EMIT vp\<%2\> = add \l" +
" EMIT vp\<%3\> = sub vp\<%2\>\l" +
" EMIT br vp\<%2\>, vp\<%3\>\l" +
"Successor(s): bb2\l"
@@ -780,7 +780,7 @@ compound=true
EXPECT_EQ(ExpectedStr, FullDump);
const char *ExpectedBlock1Str = R"(bb1:
- EMIT vp<%2> = add
+ EMIT vp<%2> = add
EMIT vp<%3> = sub vp<%2>
EMIT br vp<%2>, vp<%3>
Successor(s): bb2
@@ -842,11 +842,11 @@ TEST_F(VPBasicBlockTest, printPlanWithVFsAndUFs) {
vp<%1> = original trip-count
preheader:
- EMIT vp<%1> = sub
+ EMIT vp<%1> = sub
Successor(s): bb1
bb1:
- EMIT vp<%2> = add
+ EMIT vp<%2> = add
Successor(s): ir-bb<scalar.header>
ir-bb<scalar.header>:
@@ -866,11 +866,11 @@ No successors
vp<%1> = original trip-count
preheader:
- EMIT vp<%1> = sub
+ EMIT vp<%1> = sub
Successor(s): bb1
bb1:
- EMIT vp<%2> = add
+ EMIT vp<%2> = add
Successor(s): ir-bb<scalar.header>
ir-bb<scalar.header>:
@@ -890,11 +890,11 @@ No successors
vp<%1> = original trip-count
preheader:
- EMIT vp<%1> = sub
+ EMIT vp<%1> = sub
Successor(s): bb1
bb1:
- EMIT vp<%2> = add
+ EMIT vp<%2> = add
Successor(s): ir-bb<scalar.header>
ir-bb<scalar.header>:
@@ -932,7 +932,7 @@ compound=true
N0 -> N1 [ label=""]
N1 [label =
"bb1:\l" +
- " EMIT vp\<%1\> = add\l" +
+ " EMIT vp\<%1\> = add \l" +
" EMIT vp\<%2\> = sub vp\<%1\>\l" +
" EMIT br vp\<%1\>, vp\<%2\>\l" +
"No successors\l"
More information about the llvm-commits
mailing list