[llvm] 2befda2 - [VPlan] Populate and use VPIRFlags from initial VPInstruction. (#168450)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 18 07:15:18 PST 2025
Author: Florian Hahn
Date: 2025-11-18T15:15:14Z
New Revision: 2befda2225a6c61d0308e536c19b066ab27bbf2a
URL: https://github.com/llvm/llvm-project/commit/2befda2225a6c61d0308e536c19b066ab27bbf2a
DIFF: https://github.com/llvm/llvm-project/commit/2befda2225a6c61d0308e536c19b066ab27bbf2a.diff
LOG: [VPlan] Populate and use VPIRFlags from initial VPInstruction. (#168450)
Update VPlan to populate VPIRFlags during VPInstruction construction and
use it when creating widened recipes, instead of constructing VPIRFlags
from the underlying IR instruction each time. The VPRecipeWithIRFlags
constructor taking an underlying instruction and setting the flags based
on it has been removed.
This centralizes initial VPIRFlags creation and ensures flags are
consistently available throughout VPlan transformations and makes sure
we don't accidentally re-add flags from the underlying instruction that
already got dropped during transformations.
Follow-up to https://github.com/llvm/llvm-project/pull/167253, which did
the same for VPIRMetadata.
Should be NFC w.r.t. to the generated IR.
PR: https://github.com/llvm/llvm-project/pull/168450
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f533a47150a7b..741392247c0d6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -152,11 +152,12 @@ class VPBuilder {
/// its underlying Instruction.
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
Instruction *Inst = nullptr,
+ const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
VPInstruction *NewVPInst = tryInsertInstruction(
- new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
+ new VPInstruction(Opcode, Operands, Flags, MD, DL, Name));
NewVPInst->setUnderlyingValue(Inst);
return NewVPInst;
}
@@ -329,7 +330,7 @@ class VPBuilder {
else if (Opcode == Instruction::ZExt)
Flags = VPIRFlags::NonNegFlagsTy(false);
return tryInsertInstruction(
- new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags));
+ new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags));
}
VPScalarIVStepsRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 356d759b94799..c680b6fca84cd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7750,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
},
Range);
if (ShouldUseVectorIntrinsic)
- return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI,
+ return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, *VPI,
VPI->getDebugLoc());
Function *Variant = nullptr;
@@ -7804,7 +7804,8 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
}
Ops.push_back(VPI->getOperand(VPI->getNumOperands() - 1));
- return new VPWidenCallRecipe(CI, Variant, Ops, VPI->getDebugLoc());
+ return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI,
+ VPI->getDebugLoc());
}
return nullptr;
@@ -7842,7 +7843,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
auto *SafeRHS =
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
Ops[1] = SafeRHS;
- return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc());
+ return new VPWidenRecipe(*I, Ops, *VPI, *VPI, VPI->getDebugLoc());
}
[[fallthrough]];
}
@@ -7888,7 +7889,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
- return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
+ return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
}
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(VPI->operands());
@@ -7896,7 +7897,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getConstantInt(32, Idx));
- return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
+ return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
}
};
}
@@ -7981,7 +7982,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
auto *Recipe =
- new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI);
+ new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI,
+ *VPI, VPI->getDebugLoc());
return Recipe;
}
@@ -8231,17 +8233,19 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
return nullptr;
if (VPI->getOpcode() == Instruction::GetElementPtr)
- return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands());
+ return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands(),
+ *VPI, VPI->getDebugLoc());
if (VPI->getOpcode() == Instruction::Select)
- return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands(),
- *VPI);
+ return new VPWidenSelectRecipe(cast<SelectInst>(Instr), R->operands(), *VPI,
+ *VPI, VPI->getDebugLoc());
if (Instruction::isCast(VPI->getOpcode())) {
- auto *CastR = cast<VPInstructionWithType>(R);
auto *CI = cast<CastInst>(Instr);
+ auto *CastR = cast<VPInstructionWithType>(VPI);
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
- CastR->getResultType(), *CI, *VPI);
+ CastR->getResultType(), CI, *VPI, *VPI,
+ VPI->getDebugLoc());
}
return tryToWiden(VPI);
@@ -8269,8 +8273,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
- BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(),
- ReductionI->getDebugLoc());
+ BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRFlags(*ReductionI),
+ VPIRMetadata(), ReductionI->getDebugLoc());
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
@@ -8454,9 +8458,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
+ auto *VPI = cast<VPInstruction>(SingleDef);
auto *Recipe = new VPReplicateRecipe(
- SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
- *cast<VPInstruction>(SingleDef));
+ SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, *VPI,
+ *VPI, VPI->getDebugLoc());
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index fc29ab0c84093..fedbcfb6bd32a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -882,14 +882,6 @@ class VPIRFlags {
/// A pure-virtual common base class for recipes defining a single VPValue and
/// using IR flags.
struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- DebugLoc DL = DebugLoc::getUnknown())
- : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags() {}
-
- VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
- Instruction &I)
- : VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()), VPIRFlags(I) {}
-
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags,
DebugLoc DL = DebugLoc::getUnknown())
@@ -1474,9 +1466,12 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
VPIRMetadata(Metadata), Opcode(Opcode) {}
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
- const VPIRMetadata &Metadata, DebugLoc DL)
- : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I),
- VPIRMetadata(Metadata), Opcode(I.getOpcode()) {}
+ const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
+ DebugLoc DL = {})
+ : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
+ VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
+ setUnderlyingValue(&I);
+ }
~VPWidenRecipe() override = default;
@@ -1517,30 +1512,22 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- CastInst &UI, const VPIRMetadata &Metadata)
- : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI),
- VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
- assert(UI.getOpcode() == Opcode &&
- "opcode of underlying cast doesn't match");
- }
- VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
- const VPIRFlags &Flags = {},
+ CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
+ setUnderlyingValue(CI);
}
~VPWidenCastRecipe() override = default;
VPWidenCastRecipe *clone() override {
- auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this,
- *this, getDebugLoc());
- if (auto *UV = getUnderlyingValue())
- New->setUnderlyingValue(UV);
- return New;
+ return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
+ cast_or_null<CastInst>(getUnderlyingValue()),
+ *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
@@ -1585,13 +1572,17 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
+ const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
+ : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, Flags,
+ DL),
VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
- MayHaveSideEffects(CI.mayHaveSideEffects()) {}
+ MayHaveSideEffects(CI.mayHaveSideEffects()) {
+ setUnderlyingValue(&CI);
+ }
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
@@ -1617,7 +1608,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe *clone() override {
if (Value *CI = getUnderlyingValue())
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
- operands(), ResultTy, *this,
+ operands(), ResultTy, *this, *this,
getDebugLoc());
return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
*this, *this, getDebugLoc());
@@ -1671,10 +1662,11 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
public:
VPWidenCallRecipe(Value *UV, Function *Variant,
ArrayRef<VPValue *> CallArguments,
- DebugLoc DL = DebugLoc::getUnknown())
- : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
- *cast<Instruction>(UV)),
- VPIRMetadata(*cast<Instruction>(UV)), Variant(Variant) {
+ const VPIRFlags &Flags = {},
+ const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
+ : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, Flags, DL),
+ VPIRMetadata(Metadata), Variant(Variant) {
+ setUnderlyingValue(UV);
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
@@ -1684,7 +1676,7 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
VPWidenCallRecipe *clone() override {
return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
- getDebugLoc());
+ *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
@@ -1761,16 +1753,19 @@ class VPHistogramRecipe : public VPRecipeBase {
/// instruction.
struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
- VPWidenSelectRecipe(SelectInst &I, ArrayRef<VPValue *> Operands,
- const VPIRMetadata &MD = {})
- : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
- VPIRMetadata(MD) {}
+ VPWidenSelectRecipe(SelectInst *SI, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
+ DebugLoc DL = {})
+ : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, Flags, DL),
+ VPIRMetadata(MD) {
+ setUnderlyingValue(SI);
+ }
~VPWidenSelectRecipe() override = default;
VPWidenSelectRecipe *clone() override {
- return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
- operands(), *this);
+ return new VPWidenSelectRecipe(cast<SelectInst>(getUnderlyingInstr()),
+ operands(), *this, *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
@@ -1822,9 +1817,12 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
}
public:
- VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands)
- : VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP),
+ VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
+ const VPIRFlags &Flags = {},
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
+ setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
(void)Metadata;
getMetadataToPropagate(GEP, Metadata);
@@ -1835,7 +1833,7 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
VPWidenGEPRecipe *clone() override {
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(getUnderlyingInstr()),
- operands());
+ operands(), *this, getDebugLoc());
}
VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)
@@ -2929,10 +2927,12 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
public:
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
bool IsSingleScalar, VPValue *Mask = nullptr,
- VPIRMetadata Metadata = {})
- : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
+ const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, Flags, DL),
VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
IsPredicated(Mask) {
+ setUnderlyingValue(I);
if (Mask)
addOperand(Mask);
}
@@ -2940,9 +2940,9 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
~VPReplicateRecipe() override = default;
VPReplicateRecipe *clone() override {
- auto *Copy =
- new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsSingleScalar,
- isPredicated() ? getMask() : nullptr, *this);
+ auto *Copy = new VPReplicateRecipe(
+ getUnderlyingInstr(), operands(), IsSingleScalar,
+ isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc());
Copy->transferFlags(*this);
return Copy;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 612202d049774..dbbde1cafa9f2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -190,7 +190,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// recipes.
if (Br->isConditional()) {
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
- VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst,
+ VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, {},
VPIRMetadata(*Inst), Inst->getDebugLoc());
}
@@ -205,7 +205,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
for (auto Case : SI->cases())
Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
- VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst,
+ VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, {},
VPIRMetadata(*Inst), Inst->getDebugLoc());
continue;
}
@@ -255,13 +255,14 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
if (auto *CI = dyn_cast<CastInst>(Inst)) {
NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0],
CI->getType(), CI->getDebugLoc(),
- {}, MD);
+ VPIRFlags(*CI), MD);
NewR->setUnderlyingValue(CI);
} else {
// Build VPInstruction for any arbitrary Instruction without specific
// representation in VPlan.
- NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD,
- Inst->getDebugLoc());
+ NewR =
+ VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst,
+ VPIRFlags(*Inst), MD, Inst->getDebugLoc());
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fca6554ad77c6..ef36e29aaa5c4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2056,24 +2056,26 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
switch (OpType) {
case OperationType::OverflowingBinOp:
return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
- Opcode == Instruction::Mul ||
+ Opcode == Instruction::Mul || Opcode == Instruction::Shl ||
Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;
case OperationType::Trunc:
return Opcode == Instruction::Trunc;
case OperationType::DisjointOp:
return Opcode == Instruction::Or;
case OperationType::PossiblyExactOp:
- return Opcode == Instruction::AShr;
+ return Opcode == Instruction::AShr || Opcode == Instruction::LShr ||
+ Opcode == Instruction::UDiv || Opcode == Instruction::SDiv;
case OperationType::GEPOp:
return Opcode == Instruction::GetElementPtr ||
Opcode == VPInstruction::PtrAdd ||
Opcode == VPInstruction::WidePtrAdd;
case OperationType::FPMathOp:
- return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
- Opcode == Instruction::FSub || Opcode == Instruction::FNeg ||
- Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
- Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc ||
- Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
+ return Opcode == Instruction::Call || Opcode == Instruction::FAdd ||
+ Opcode == Instruction::FMul || Opcode == Instruction::FSub ||
+ Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||
+ Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||
+ Opcode == Instruction::FPTrunc || Opcode == Instruction::FCmp ||
+ Opcode == Instruction::Select ||
Opcode == VPInstruction::WideIVStep ||
Opcode == VPInstruction::ReductionStartVector ||
Opcode == VPInstruction::ComputeReductionResult;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 26563242de283..25557f1d5d651 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -104,24 +104,26 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, *VPI,
Ingredient.getDebugLoc());
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
- NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
+ NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands(), *VPI,
+ Ingredient.getDebugLoc());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI);
if (VectorID == Intrinsic::not_intrinsic)
return false;
NewRecipe = new VPWidenIntrinsicRecipe(
*CI, getVectorIntrinsicIDForCall(CI, &TLI),
- drop_end(Ingredient.operands()), CI->getType(), *VPI,
- CI->getDebugLoc());
+ drop_end(Ingredient.operands()), CI->getType(), VPIRFlags(*CI),
+ *VPI, CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
- NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands(), *VPI);
+ NewRecipe = new VPWidenSelectRecipe(SI, Ingredient.operands(), *VPI,
+ *VPI, Ingredient.getDebugLoc());
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
- NewRecipe =
- new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0),
- CI->getType(), *CI, *VPI);
+ NewRecipe = new VPWidenCastRecipe(
+ CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), CI,
+ VPIRFlags(*CI), VPIRMetadata(*CI));
} else {
NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
- Ingredient.getDebugLoc());
+ *VPI, Ingredient.getDebugLoc());
}
}
@@ -226,7 +228,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
// then cloning should be sufficient here.
Instruction *I = SinkCandidate->getUnderlyingInstr();
Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true,
- nullptr /*Mask*/, *SinkCandidateRepR);
+ nullptr /*Mask*/, *SinkCandidateRepR,
+ *SinkCandidateRepR);
// TODO: add ".cloned" suffix to name of Clone's VPValue.
} else {
Clone = SinkCandidate->clone();
@@ -385,7 +388,8 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
// mask but in the replicate region.
auto *RecipeWithoutMask = new VPReplicateRecipe(
PredRecipe->getUnderlyingInstr(), drop_end(PredRecipe->operands()),
- PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe);
+ PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
+ PredRecipe->getDebugLoc());
auto *Pred =
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
@@ -691,7 +695,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
// analysis.
auto Users = collectUsersRecursively(PhiR);
for (VPUser *U : reverse(Users)) {
- auto *Def = dyn_cast<VPSingleDefRecipe>(U);
+ auto *Def = dyn_cast<VPRecipeWithIRFlags>(U);
auto *RepR = dyn_cast<VPReplicateRecipe>(U);
// Skip recipes that shouldn't be narrowed.
if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
@@ -704,7 +708,8 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
continue;
auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(),
- Def->operands(), /*IsUniform*/ true);
+ Def->operands(), /*IsUniform*/ true,
+ /*Mask*/ nullptr, /*Flags*/ *Def);
Clone->insertAfter(Def);
Def->replaceAllUsesWith(Clone);
}
@@ -1423,12 +1428,13 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
continue;
- auto *RepOrWidenR = cast<VPSingleDefRecipe>(&R);
+ auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(&R);
if (RepR && isa<StoreInst>(RepR->getUnderlyingInstr()) &&
vputils::isSingleScalar(RepR->getOperand(1))) {
auto *Clone = new VPReplicateRecipe(
RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
- true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Metadata*/);
+ true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
+ *RepR /*Metadata*/, RepR->getDebugLoc());
Clone->insertBefore(RepOrWidenR);
unsigned ExtractOpc =
vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1))
@@ -1469,9 +1475,9 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
}))
continue;
- auto *Clone = new VPReplicateRecipe(RepOrWidenR->getUnderlyingInstr(),
- RepOrWidenR->operands(),
- true /*IsSingleScalar*/);
+ auto *Clone = new VPReplicateRecipe(
+ RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
+ true /*IsSingleScalar*/, nullptr, *RepOrWidenR);
Clone->insertBefore(RepOrWidenR);
RepOrWidenR->replaceAllUsesWith(Clone);
if (isDeadRecipe(*RepOrWidenR))
@@ -3824,15 +3830,15 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
Ext0->getOpcode() == Ext1->getOpcode() &&
IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
auto *NewExt0 = new VPWidenCastRecipe(
- Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0,
- *Ext0, Ext0->getDebugLoc());
+ Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), nullptr,
+ *Ext0, *Ext0, Ext0->getDebugLoc());
NewExt0->insertBefore(Ext0);
VPWidenCastRecipe *NewExt1 = NewExt0;
if (Ext0 != Ext1) {
NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0),
- Ext->getResultType(), *Ext1, *Ext1,
- Ext1->getDebugLoc());
+ Ext->getResultType(), nullptr, *Ext1,
+ *Ext1, Ext1->getDebugLoc());
NewExt1->insertBefore(Ext1);
}
Mul->setOperand(0, NewExt0);
@@ -4353,7 +4359,7 @@ narrowInterleaveGroupOp(VPValue *V, SmallPtrSetImpl<VPValue *> &NarrowedOps) {
// process one original iteration.
auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
/*IsUniform*/ true,
- /*Mask*/ nullptr, *WideLoad);
+ /*Mask*/ nullptr, {}, *WideLoad);
N->insertBefore(WideLoad);
NarrowedOps.insert(N);
return N;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index d4b8b72beb942..d76d2ed5f1c76 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -518,9 +518,9 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
// TODO: have cloning of replicate recipes also provide the desired result
// coupled with setting its operands to NewOps (deriving IsSingleScalar and
// Mask from the operands?)
- New =
- new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps,
- /*IsSingleScalar=*/true, /*Mask=*/nullptr, *RepR);
+ New = new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps,
+ /*IsSingleScalar=*/true, /*Mask=*/nullptr,
+ *RepR, *RepR, RepR->getDebugLoc());
} else {
assert(isa<VPInstruction>(DefR) &&
"DefR must be a VPReplicateRecipe or VPInstruction");
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 20676f3702294..10c265519952b 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -14,23 +14,23 @@ define void @foo(i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: outer.header:
; CHECK-NEXT: EMIT-SCALAR ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
+; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr inbounds ir<@arr2>, ir<0>, ir<%outer.iv>
; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1>
-; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
+; CHECK-NEXT: EMIT ir<%add> = add nsw ir<%outer.iv>, ir<%n>
; CHECK-NEXT: Successor(s): inner
; CHECK-EMPTY:
; CHECK-NEXT: inner:
; CHECK-NEXT: EMIT-SCALAR ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ]
-; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
+; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr inbounds ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2>
-; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8>
+; CHECK-NEXT: EMIT ir<%inner.iv.next> = add nuw nsw ir<%inner.iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%inner.ec> = icmp eq ir<%inner.iv.next>, ir<8>
; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
; CHECK-NEXT: Successor(s): outer.latch, inner
; CHECK-EMPTY:
; CHECK-NEXT: outer.latch:
-; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8>
+; CHECK-NEXT: EMIT ir<%outer.iv.next> = add nuw nsw ir<%outer.iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%outer.ec> = icmp eq ir<%outer.iv.next>, ir<8>
; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
; CHECK-NEXT: Successor(s): ir-bb<exit>, outer.header
; CHECK-EMPTY:
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index b99d656c5c50f..5742df2aa3c53 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -139,12 +139,12 @@ compound=true
"vector.body:\l" +
" EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" +
" EMIT-SCALAR ir\<%indvars.iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%indvars.iv.next\>, vector.body ]\l" +
- " EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%indvars.iv\>\l" +
+ " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%indvars.iv\>\l" +
" EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
" EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" +
" EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
" EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\>, ir\<1\>\l" +
- " EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\>, ir\<%N\>\l" +
+ " EMIT ir\<%exitcond\> = icmp ne ir\<%indvars.iv.next\>, ir\<%N\>\l" +
" EMIT vp\<%3\> = not ir\<%exitcond\>\l" +
" EMIT vp\<%index.next\> = add nuw vp\<%2\>, vp\<%0\>\l" +
" EMIT branch-on-count vp\<%index.next\>, vp\<%1\>\l" +
@@ -305,9 +305,9 @@ compound=true
"vector.body:\l" +
" EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" +
" EMIT-SCALAR ir\<%iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%iv.next\>, loop.latch ]\l" +
- " EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%iv\>\l" +
+ " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%iv\>\l" +
" EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
- " EMIT ir\<%c\> = icmp ir\<%l1\>, ir\<0\>\l" +
+ " EMIT ir\<%c\> = icmp eq ir\<%l1\>, ir\<0\>\l" +
"Successor(s): loop.latch\l"
]
N4 -> N6 [ label=""]
@@ -316,7 +316,7 @@ compound=true
" EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" +
" EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
" EMIT ir\<%iv.next\> = add ir\<%iv\>, ir\<1\>\l" +
- " EMIT ir\<%exitcond\> = icmp ir\<%iv.next\>, ir\<%N\>\l" +
+ " EMIT ir\<%exitcond\> = icmp ne ir\<%iv.next\>, ir\<%N\>\l" +
" EMIT vp\<%3\> = not ir\<%exitcond\>\l" +
" EMIT vp\<%index.next\> = add nuw vp\<%2\>, vp\<%0\>\l" +
" EMIT branch-on-count vp\<%index.next\>, vp\<%1\>\l" +
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 3842ba235ead3..63776b78a2088 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1009,7 +1009,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe WidenR(*AI, Args, VPIRMetadata(), DebugLoc());
+ VPWidenRecipe WidenR(*AI, Args);
checkVPRecipeCastImpl<VPWidenRecipe, VPUser, VPIRMetadata>(&WidenR);
delete AI;
@@ -1053,7 +1053,7 @@ TEST_F(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(Op3);
- VPWidenSelectRecipe WidenSelectR(*SelectI,
+ VPWidenSelectRecipe WidenSelectR(SelectI,
make_range(Args.begin(), Args.end()));
checkVPRecipeCastImpl<VPWidenSelectRecipe, VPUser, VPIRMetadata>(
@@ -1093,7 +1093,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) {
IntegerType *Int64 = IntegerType::get(C, 64);
auto *Cast = CastInst::CreateZExtOrBitCast(PoisonValue::get(Int32), Int64);
VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
- VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast, {});
+ VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, Cast);
checkVPRecipeCastImpl<VPWidenCastRecipe, VPUser, VPIRMetadata>(&Recipe);
delete Cast;
@@ -1264,7 +1264,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
SmallVector<VPValue *, 2> Args;
Args.push_back(Op1);
Args.push_back(Op2);
- VPWidenRecipe Recipe(*AI, Args, VPIRMetadata(), DebugLoc());
+ VPWidenRecipe Recipe(*AI, Args);
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1283,7 +1283,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(Op3);
- VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end()));
+ VPWidenSelectRecipe Recipe(SelectI, make_range(Args.begin(), Args.end()));
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1412,7 +1412,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
Args.push_back(Op1);
Args.push_back(Op2);
Args.push_back(CalledFn);
- VPWidenCallRecipe Recipe(Call, TheFn, Args);
+ VPWidenCallRecipe Recipe(Call, TheFn, Args, VPIRFlags(), VPIRMetadata());
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
@@ -1468,8 +1468,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) {
VPValue *ExtVPV2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
Args.push_back(ExtVPV1);
Args.push_back(ExtVPV2);
- VPWidenRecipe *WidenR =
- new VPWidenRecipe(*AI, Args, VPIRMetadata(), DebugLoc());
+ VPWidenRecipe *WidenR = new VPWidenRecipe(*AI, Args);
VPBB1->appendRecipe(WidenR);
{
More information about the llvm-commits
mailing list