[llvm] 8a6d5f6 - [VPlan] Update more VPRecipeBuilder members to take VPInst directly (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Nov 7 13:06:47 PST 2025
Author: Florian Hahn
Date: 2025-11-07T21:06:38Z
New Revision: 8a6d5f68e48f3a20effd2bf78a99130b89ca413e
URL: https://github.com/llvm/llvm-project/commit/8a6d5f68e48f3a20effd2bf78a99130b89ca413e
DIFF: https://github.com/llvm/llvm-project/commit/8a6d5f68e48f3a20effd2bf78a99130b89ca413e.diff
LOG: [VPlan] Update more VPRecipeBuilder members to take VPInst directly (NFC)
Update VPRecipeBuilder methos to take VPInstruction* directly instead of
ArrayRef<> for operands and Instruction * separately.
This allows avoid accessing the underlying instruction in some cases, by
using information directly from VPInstruction, like getOpcode(),
getDebugLoc(), and getOperand().
It also allows directly transferring other information directly from
VPInstruction in the future.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b7224a33f47b1..666033b1fac62 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7488,11 +7488,12 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
});
}
-VPWidenMemoryRecipe *
-VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
- VFRange &Range) {
- assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
+VPWidenMemoryRecipe *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
+ VFRange &Range) {
+ assert((VPI->getOpcode() == Instruction::Load ||
+ VPI->getOpcode() == Instruction::Store) &&
"Must be called with either a load or store");
+ Instruction *I = VPI->getUnderlyingInstr();
auto WillWiden = [&](ElementCount VF) -> bool {
LoopVectorizationCostModel::InstWidening Decision =
@@ -7522,7 +7523,8 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
bool Consecutive =
Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
- VPValue *Ptr = isa<LoadInst>(I) ? Operands[0] : Operands[1];
+ VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
+ : VPI->getOperand(1);
if (Consecutive) {
auto *GEP = dyn_cast<GetElementPtrInst>(
Ptr->getUnderlyingValue()->stripPointerCasts());
@@ -7536,77 +7538,78 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
CM.foldTailByMasking() || !GEP
? GEPNoWrapFlags::none()
: GEP->getNoWrapFlags().withoutNoUnsignedWrap();
- VectorPtr =
- new VPVectorEndPointerRecipe(Ptr, &Plan.getVF(), getLoadStoreType(I),
- /*Stride*/ -1, Flags, I->getDebugLoc());
+ VectorPtr = new VPVectorEndPointerRecipe(
+ Ptr, &Plan.getVF(), getLoadStoreType(I),
+ /*Stride*/ -1, Flags, VPI->getDebugLoc());
} else {
VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
GEP ? GEP->getNoWrapFlags()
: GEPNoWrapFlags::none(),
- I->getDebugLoc());
+ VPI->getDebugLoc());
}
Builder.insert(VectorPtr);
Ptr = VectorPtr;
}
- if (LoadInst *Load = dyn_cast<LoadInst>(I))
+ if (VPI->getOpcode() == Instruction::Load) {
+ auto *Load = cast<LoadInst>(I);
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
VPIRMetadata(*Load, LVer), I->getDebugLoc());
+ }
StoreInst *Store = cast<StoreInst>(I);
- return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
- Reverse, VPIRMetadata(*Store, LVer),
- I->getDebugLoc());
+ return new VPWidenStoreRecipe(*Store, Ptr, VPI->getOperand(0), Mask,
+ Consecutive, Reverse,
+ VPIRMetadata(*Store, LVer), VPI->getDebugLoc());
}
-/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
-/// insert a recipe to expand the step for the induction recipe.
+/// Creates a VPWidenIntOrFpInductionRecipe for \p PhiR. If needed, it will
+/// also insert a recipe to expand the step for the induction recipe.
static VPWidenIntOrFpInductionRecipe *
-createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc,
- VPValue *Start, const InductionDescriptor &IndDesc,
- VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop) {
- assert(IndDesc.getStartValue() ==
- Phi->getIncomingValueForBlock(OrigLoop.getLoopPreheader()));
+createWidenInductionRecipes(VPInstruction *PhiR,
+ const InductionDescriptor &IndDesc, VPlan &Plan,
+ ScalarEvolution &SE, Loop &OrigLoop) {
assert(SE.isLoopInvariant(IndDesc.getStep(), &OrigLoop) &&
"step must be loop invariant");
+ VPValue *Start = PhiR->getOperand(0);
+ assert(Plan.getLiveIn(IndDesc.getStartValue()) == Start &&
+ "Start VPValue must match IndDesc's start value");
+
VPValue *Step =
vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
- if (auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
- return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
- IndDesc, TruncI,
- TruncI->getDebugLoc());
- }
- assert(isa<PHINode>(PhiOrTrunc) && "must be a phi node here");
+ PHINode *Phi = cast<PHINode>(PhiR->getUnderlyingInstr());
return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
- IndDesc, Phi->getDebugLoc());
+ IndDesc, PhiR->getDebugLoc());
}
-VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
- PHINode *Phi, ArrayRef<VPValue *> Operands, VFRange &Range) {
+VPHeaderPHIRecipe *
+VPRecipeBuilder::tryToOptimizeInductionPHI(VPInstruction *VPI, VFRange &Range) {
+ auto *Phi = cast<PHINode>(VPI->getUnderlyingInstr());
// Check if this is an integer or fp induction. If so, build the recipe that
// produces its scalar and vector values.
if (auto *II = Legal->getIntOrFpInductionDescriptor(Phi))
- return createWidenInductionRecipes(Phi, Phi, Operands[0], *II, Plan,
- *PSE.getSE(), *OrigLoop);
+ return createWidenInductionRecipes(VPI, *II, Plan, *PSE.getSE(), *OrigLoop);
// Check if this is pointer induction. If so, build the recipe for it.
if (auto *II = Legal->getPointerInductionDescriptor(Phi)) {
VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep());
return new VPWidenPointerInductionRecipe(
- Phi, Operands[0], Step, &Plan.getVFxUF(), *II,
+ Phi, VPI->getOperand(0), Step, &Plan.getVFxUF(), *II,
LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) {
return CM.isScalarAfterVectorization(Phi, VF);
},
Range),
- Phi->getDebugLoc());
+ VPI->getDebugLoc());
}
return nullptr;
}
-VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
- TruncInst *I, ArrayRef<VPValue *> Operands, VFRange &Range) {
+VPWidenIntOrFpInductionRecipe *
+VPRecipeBuilder::tryToOptimizeInductionTruncate(VPInstruction *VPI,
+ VFRange &Range) {
+ auto *I = cast<TruncInst>(VPI->getUnderlyingInstr());
// Optimize the special case where the source is a constant integer
// induction variable. Notice that we can only optimize the 'trunc' case
// because (a) FP conversions lose precision, (b) sext/zext may wrap, and
@@ -7621,21 +7624,24 @@ VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
};
};
- if (LoopVectorizationPlanner::getDecisionAndClampRange(
- IsOptimizableIVTruncate(I), Range)) {
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ IsOptimizableIVTruncate(I), Range))
+ return nullptr;
- auto *Phi = cast<PHINode>(I->getOperand(0));
- const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi);
- VPValue *Start = Plan.getOrAddLiveIn(II.getStartValue());
- return createWidenInductionRecipes(Phi, I, Start, II, Plan, *PSE.getSE(),
- *OrigLoop);
- }
- return nullptr;
+ auto *WidenIV = cast<VPWidenIntOrFpInductionRecipe>(
+ VPI->getOperand(0)->getDefiningRecipe());
+ PHINode *Phi = WidenIV->getPHINode();
+ VPValue *Start = WidenIV->getStartValue();
+ const InductionDescriptor &IndDesc = WidenIV->getInductionDescriptor();
+ VPValue *Step =
+ vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
+ return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
+ IndDesc, I, VPI->getDebugLoc());
}
-VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
- ArrayRef<VPValue *> Operands,
+VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
VFRange &Range) {
+ CallInst *CI = cast<CallInst>(VPI->getUnderlyingInstr());
bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
[this, CI](ElementCount VF) {
return CM.isScalarWithPredication(CI, VF);
@@ -7652,7 +7658,8 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
ID == Intrinsic::experimental_noalias_scope_decl))
return nullptr;
- SmallVector<VPValue *, 4> Ops(Operands.take_front(CI->arg_size()));
+ SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
+ VPI->op_begin() + CI->arg_size());
// Is it beneficial to perform intrinsic call compared to lib call?
bool ShouldUseVectorIntrinsic =
@@ -7664,7 +7671,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
Range);
if (ShouldUseVectorIntrinsic)
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
- CI->getDebugLoc());
+ VPI->getDebugLoc());
Function *Variant = nullptr;
std::optional<unsigned> MaskPos;
@@ -7711,13 +7718,13 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
Mask = getBlockInMask(Builder.getInsertBlock());
else
Mask = Plan.getOrAddLiveIn(
- ConstantInt::getTrue(IntegerType::getInt1Ty(CI->getContext())));
+ ConstantInt::getTrue(IntegerType::getInt1Ty(Plan.getContext())));
Ops.insert(Ops.begin() + *MaskPos, Mask);
}
- Ops.push_back(Operands.back());
- return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc());
+ Ops.push_back(VPI->getOperand(VPI->getNumOperands() - 1));
+ return new VPWidenCallRecipe(CI, Variant, Ops, VPI->getDebugLoc());
}
return nullptr;
@@ -7737,9 +7744,9 @@ bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
Range);
}
-VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
- ArrayRef<VPValue *> Operands) {
- switch (I->getOpcode()) {
+VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
+ auto *I = VPI->getUnderlyingInstr();
+ switch (VPI->getOpcode()) {
default:
return nullptr;
case Instruction::SDiv:
@@ -7749,10 +7756,11 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
// If not provably safe, use a select to form a safe divisor before widening the
// div/rem operation itself. Otherwise fall through to general handling below.
if (CM.isPredicatedInst(I)) {
- SmallVector<VPValue *> Ops(Operands);
+ SmallVector<VPValue *> Ops(VPI->operands());
VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
VPValue *One = Plan.getConstantInt(I->getType(), 1u);
- auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
+ auto *SafeRHS =
+ Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
Ops[1] = SafeRHS;
return new VPWidenRecipe(*I, Ops);
}
@@ -7777,8 +7785,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
case Instruction::Sub:
case Instruction::Xor:
case Instruction::Freeze: {
- SmallVector<VPValue *> NewOps(Operands);
- if (Instruction::isBinaryOp(I->getOpcode())) {
+ SmallVector<VPValue *> NewOps(VPI->operands());
+ if (Instruction::isBinaryOp(VPI->getOpcode())) {
// The legacy cost model uses SCEV to check if some of the operands are
// constants. To match the legacy cost model's behavior, use SCEV to try
// to replace operands with constants.
@@ -7795,7 +7803,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
return Plan.getOrAddLiveIn(C->getValue());
};
// For Mul, the legacy cost model checks both operands.
- if (I->getOpcode() == Instruction::Mul)
+ if (VPI->getOpcode() == Instruction::Mul)
NewOps[0] = GetConstantViaSCEV(NewOps[0]);
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
@@ -7803,7 +7811,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
return new VPWidenRecipe(*I, NewOps);
}
case Instruction::ExtractValue: {
- SmallVector<VPValue *> NewOps(Operands);
+ SmallVector<VPValue *> NewOps(VPI->operands());
auto *EVI = cast<ExtractValueInst>(I);
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
@@ -7813,9 +7821,8 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
};
}
-VPHistogramRecipe *
-VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
- ArrayRef<VPValue *> Operands) {
+VPHistogramRecipe *VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
+ VPInstruction *VPI) {
// FIXME: Support other operations.
unsigned Opcode = HI->Update->getOpcode();
assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
@@ -7823,7 +7830,7 @@ VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
SmallVector<VPValue *, 3> HGramOps;
// Bucket address.
- HGramOps.push_back(Operands[1]);
+ HGramOps.push_back(VPI->getOperand(1));
// Increment value.
HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1)));
@@ -7832,12 +7839,12 @@ VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
if (Legal->isMaskRequired(HI->Store))
HGramOps.push_back(getBlockInMask(Builder.getInsertBlock()));
- return new VPHistogramRecipe(Opcode, HGramOps, HI->Store->getDebugLoc());
+ return new VPHistogramRecipe(Opcode, HGramOps, VPI->getDebugLoc());
}
-VPReplicateRecipe *
-VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
- VFRange &Range) {
+VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
+ VFRange &Range) {
+ auto *I = VPI->getUnderlyingInstr();
bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
Range);
@@ -7893,8 +7900,8 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
- auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
- VPIRMetadata(*I, LVer));
+ auto *Recipe = new VPReplicateRecipe(I, VPI->operands(), IsUniform,
+ BlockInMask, VPIRMetadata(*I, LVer));
return Recipe;
}
@@ -8075,8 +8082,6 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
// First, check for specific widening recipes that deal with inductions, Phi
// nodes, calls and memory operations.
VPRecipeBase *Recipe;
- Instruction *Instr = R->getUnderlyingInstr();
- SmallVector<VPValue *, 4> Operands(R->operands());
if (auto *PhiR = dyn_cast<VPPhi>(R)) {
VPBasicBlock *Parent = PhiR->getParent();
[[maybe_unused]] VPRegionBlock *LoopRegionOf =
@@ -8084,15 +8089,15 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
assert(LoopRegionOf && LoopRegionOf->getEntry() == Parent &&
"Non-header phis should have been handled during predication");
auto *Phi = cast<PHINode>(R->getUnderlyingInstr());
- assert(Operands.size() == 2 && "Must have 2 operands for header phis");
- if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
+ assert(R->getNumOperands() == 2 && "Must have 2 operands for header phis");
+ if ((Recipe = tryToOptimizeInductionPHI(PhiR, Range)))
return Recipe;
VPHeaderPHIRecipe *PhiRecipe = nullptr;
assert((Legal->isReductionVariable(Phi) ||
Legal->isFixedOrderRecurrence(Phi)) &&
"can only widen reductions and fixed-order recurrences here");
- VPValue *StartV = Operands[0];
+ VPValue *StartV = R->getOperand(0);
if (Legal->isReductionVariable(Phi)) {
const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
assert(RdxDesc.getRecurrenceStartValue() ==
@@ -8112,13 +8117,15 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV);
}
// Add backedge value.
- PhiRecipe->addOperand(Operands[1]);
+ PhiRecipe->addOperand(R->getOperand(1));
return PhiRecipe;
}
assert(!R->isPhi() && "only VPPhi nodes expected at this point");
- if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
- cast<TruncInst>(Instr), Operands, Range)))
+ auto *VPI = cast<VPInstruction>(R);
+ Instruction *Instr = R->getUnderlyingInstr();
+ if (VPI->getOpcode() == Instruction::Trunc &&
+ (Recipe = tryToOptimizeInductionTruncate(VPI, Range)))
return Recipe;
// All widen recipes below deal only with VF > 1.
@@ -8126,46 +8133,46 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
[&](ElementCount VF) { return VF.isScalar(); }, Range))
return nullptr;
- if (auto *CI = dyn_cast<CallInst>(Instr))
- return tryToWidenCall(CI, Operands, Range);
+ if (VPI->getOpcode() == Instruction::Call)
+ return tryToWidenCall(VPI, Range);
- if (StoreInst *SI = dyn_cast<StoreInst>(Instr))
- if (auto HistInfo = Legal->getHistogramInfo(SI))
- return tryToWidenHistogram(*HistInfo, Operands);
+ if (VPI->getOpcode() == Instruction::Store)
+ if (auto HistInfo = Legal->getHistogramInfo(cast<StoreInst>(Instr)))
+ return tryToWidenHistogram(*HistInfo, VPI);
- if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
- return tryToWidenMemory(Instr, Operands, Range);
+ if (VPI->getOpcode() == Instruction::Load ||
+ VPI->getOpcode() == Instruction::Store)
+ return tryToWidenMemory(VPI, Range);
if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr))
- return tryToCreatePartialReduction(Instr, Operands, ScaleFactor.value());
+ return tryToCreatePartialReduction(VPI, ScaleFactor.value());
if (!shouldWiden(Instr, Range))
return nullptr;
- if (auto *GEP = dyn_cast<GetElementPtrInst>(Instr))
- return new VPWidenGEPRecipe(GEP, Operands);
+ if (VPI->getOpcode() == Instruction::GetElementPtr)
+ return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands());
- if (auto *SI = dyn_cast<SelectInst>(Instr)) {
- return new VPWidenSelectRecipe(*SI, Operands);
- }
+ if (VPI->getOpcode() == Instruction::Select)
+ return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands());
- if (auto *CI = dyn_cast<CastInst>(Instr)) {
- return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
- *CI);
+ if (Instruction::isCast(VPI->getOpcode())) {
+ auto *CI = cast<CastInst>(Instr);
+ return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
+ CI->getType(), *CI);
}
- return tryToWiden(Instr, Operands);
+ return tryToWiden(VPI);
}
VPRecipeBase *
-VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
- ArrayRef<VPValue *> Operands,
+VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
unsigned ScaleFactor) {
- assert(Operands.size() == 2 &&
+ assert(Reduction->getNumOperands() == 2 &&
"Unexpected number of operands for partial reduction");
- VPValue *BinOp = Operands[0];
- VPValue *Accumulator = Operands[1];
+ VPValue *BinOp = Reduction->getOperand(0);
+ VPValue *Accumulator = Reduction->getOperand(1);
VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
isa<VPPartialReductionRecipe>(BinOpRecipe))
@@ -8176,28 +8183,29 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
"all accumulators in chain must have same scale factor");
unsigned ReductionOpcode = Reduction->getOpcode();
+ auto *ReductionI = Reduction->getUnderlyingInstr();
if (ReductionOpcode == Instruction::Sub) {
- auto *const Zero = ConstantInt::get(Reduction->getType(), 0);
+ auto *const Zero = ConstantInt::get(ReductionI->getType(), 0);
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
- BinOp = new VPWidenRecipe(*Reduction, Ops);
+ BinOp = new VPWidenRecipe(*ReductionI, Ops);
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
VPValue *Cond = nullptr;
- if (CM.blockNeedsPredicationForAnyReason(Reduction->getParent())) {
+ if (CM.blockNeedsPredicationForAnyReason(ReductionI->getParent())) {
assert((ReductionOpcode == Instruction::Add ||
ReductionOpcode == Instruction::Sub) &&
"Expected an ADD or SUB operation for predicated partial "
"reductions (because the neutral element in the mask is zero)!");
Cond = getBlockInMask(Builder.getInsertBlock());
- VPValue *Zero = Plan.getConstantInt(Reduction->getType(), 0);
+ VPValue *Zero = Plan.getConstantInt(ReductionI->getType(), 0);
BinOp = Builder.createSelect(Cond, BinOp, Zero, Reduction->getDebugLoc());
}
return new VPPartialReductionRecipe(ReductionOpcode, Accumulator, BinOp, Cond,
- ScaleFactor, Reduction);
+ ScaleFactor, ReductionI);
}
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
@@ -8382,7 +8390,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
VPRecipeBase *Recipe =
RecipeBuilder.tryToCreateWidenRecipe(SingleDef, Range);
if (!Recipe)
- Recipe = RecipeBuilder.handleReplication(Instr, R.operands(), Range);
+ Recipe = RecipeBuilder.handleReplication(cast<VPInstruction>(SingleDef),
+ Range);
RecipeBuilder.setRecipe(Instr, Recipe);
if (isa<VPWidenIntOrFpInductionRecipe>(Recipe) && isa<TruncInst>(Instr)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 41878e3c648e3..a7000aff06379 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -93,42 +93,37 @@ class VPRecipeBuilder {
/// Range. The function should not be called for memory instructions or calls.
bool shouldWiden(Instruction *I, VFRange &Range) const;
- /// Check if the load or store instruction \p I should widened for \p
+ /// Check if the load or store instruction \p VPI should widened for \p
/// Range.Start and potentially masked. Such instructions are handled by a
/// recipe that takes an additional VPInstruction for the mask.
- VPWidenMemoryRecipe *tryToWidenMemory(Instruction *I,
- ArrayRef<VPValue *> Operands,
- VFRange &Range);
+ VPWidenMemoryRecipe *tryToWidenMemory(VPInstruction *VPI, VFRange &Range);
- /// Check if an induction recipe should be constructed for \p Phi. If so build
+ /// Check if an induction recipe should be constructed for \p VPI. If so build
/// and return it. If not, return null.
- VPHeaderPHIRecipe *tryToOptimizeInductionPHI(PHINode *Phi,
- ArrayRef<VPValue *> Operands,
+ VPHeaderPHIRecipe *tryToOptimizeInductionPHI(VPInstruction *VPI,
VFRange &Range);
- /// Optimize the special case where the operand of \p I is a constant integer
- /// induction variable.
+ /// Optimize the special case where the operand of \p VPI is a constant
+ /// integer induction variable.
VPWidenIntOrFpInductionRecipe *
- tryToOptimizeInductionTruncate(TruncInst *I, ArrayRef<VPValue *> Operands,
- VFRange &Range);
+ tryToOptimizeInductionTruncate(VPInstruction *VPI, VFRange &Range);
- /// Handle call instructions. If \p CI can be widened for \p Range.Start,
+ /// Handle call instructions. If \p VPI can be widened for \p Range.Start,
/// return a new VPWidenCallRecipe or VPWidenIntrinsicRecipe. Range.End may be
/// decreased to ensure same decision from \p Range.Start to \p Range.End.
- VPSingleDefRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
- VFRange &Range);
+ VPSingleDefRecipe *tryToWidenCall(VPInstruction *VPI, VFRange &Range);
- /// Check if \p I has an opcode that can be widened and return a VPWidenRecipe
- /// if it can. The function should only be called if the cost-model indicates
- /// that widening should be performed.
- VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands);
+ /// Check if \p VPI has an opcode that can be widened and return a
+ /// VPWidenRecipe if it can. The function should only be called if the
+ /// cost-model indicates that widening should be performed.
+ VPWidenRecipe *tryToWiden(VPInstruction *VPI);
/// Makes Histogram count operations safe for vectorization, by emitting a
/// llvm.experimental.vector.histogram.add intrinsic in place of the
/// Load + Add|Sub + Store operations that perform the histogram in the
/// original scalar loop.
VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
- ArrayRef<VPValue *> Operands);
+ VPInstruction *VPI);
/// Examines reduction operations to see if the target can use a cheaper
/// operation with a wider per-iteration input VF and narrower PHI VF.
@@ -171,8 +166,7 @@ class VPRecipeBuilder {
/// Create and return a partial reduction recipe for a reduction instruction
/// along with binary operation and reduction phi operands.
- VPRecipeBase *tryToCreatePartialReduction(Instruction *Reduction,
- ArrayRef<VPValue *> Operands,
+ VPRecipeBase *tryToCreatePartialReduction(VPInstruction *Reduction,
unsigned ScaleFactor);
/// Set the recipe created for given ingredient.
@@ -197,12 +191,10 @@ class VPRecipeBuilder {
return Ingredient2Recipe[I];
}
- /// Build a VPReplicationRecipe for \p I using \p Operands. If it is
- /// predicated, add the mask as last operand. Range.End may be decreased to
- /// ensure same recipe behavior from \p Range.Start to \p Range.End.
- VPReplicateRecipe *handleReplication(Instruction *I,
- ArrayRef<VPValue *> Operands,
- VFRange &Range);
+ /// Build a VPReplicationRecipe for \p VPI. If it is predicated, add the mask
+ /// as last operand. Range.End may be decreased to ensure same recipe behavior
+ /// from \p Range.Start to \p Range.End.
+ VPReplicateRecipe *handleReplication(VPInstruction *VPI, VFRange &Range);
VPValue *getVPValueOrAddLiveIn(Value *V) {
if (auto *I = dyn_cast<Instruction>(V)) {
More information about the llvm-commits
mailing list