[llvm] 408ebe5 - [VPlan] Move VPWidenCallRecipe to VPlanRecipes.cpp (NFC).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 5 02:48:53 PDT 2022
Author: Florian Hahn
Date: 2022-09-05T10:48:29+01:00
New Revision: 408ebe5e3aae3cfa1e960f74998b02e277591663
URL: https://github.com/llvm/llvm-project/commit/408ebe5e3aae3cfa1e960f74998b02e277591663
DIFF: https://github.com/llvm/llvm-project/commit/408ebe5e3aae3cfa1e960f74998b02e277591663.diff
LOG: [VPlan] Move VPWidenCallRecipe to VPlanRecipes.cpp (NFC).
Depends on D132585.
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D132586
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5b509250a874b..5448c3131da81 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4153,63 +4153,6 @@ bool InnerLoopVectorizer::useOrderedReductions(
return Cost->useOrderedReductions(RdxDesc);
}
-void InnerLoopVectorizer::widenCallInstruction(
- CallInst &CI, VPValue *Def, VPUser &ArgOperands, VPTransformState &State,
- Intrinsic::ID VectorIntrinsicID) {
- assert(!isa<DbgInfoIntrinsic>(CI) &&
- "DbgInfoIntrinsic should have been dropped during VPlan construction");
- State.setDebugLocFromInst(&CI);
-
- SmallVector<Type *, 4> Tys;
- for (Value *ArgOperand : CI.args())
- Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));
-
- for (unsigned Part = 0; Part < UF; ++Part) {
- SmallVector<Type *, 2> TysForDecl = {CI.getType()};
- SmallVector<Value *, 4> Args;
- for (const auto &I : enumerate(ArgOperands.operands())) {
- // Some intrinsics have a scalar argument - don't replace it with a
- // vector.
- Value *Arg;
- if (!VectorIntrinsicID ||
- !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
- Arg = State.get(I.value(), Part);
- else
- Arg = State.get(I.value(), VPIteration(0, 0));
- if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
- TysForDecl.push_back(Arg->getType());
- Args.push_back(Arg);
- }
-
- Function *VectorF;
- if (VectorIntrinsicID) {
- // Use vector version of the intrinsic.
- if (VF.isVector())
- TysForDecl[0] = VectorType::get(CI.getType()->getScalarType(), VF);
- Module *M = State.Builder.GetInsertBlock()->getModule();
- VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
- assert(VectorF && "Can't retrieve vector intrinsic.");
- } else {
- // Use vector version of the function call.
- const VFShape Shape = VFShape::get(CI, VF, false /*HasGlobalPred*/);
-#ifndef NDEBUG
- assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
- "Can't create vector function.");
-#endif
- VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
- }
- SmallVector<OperandBundleDef, 1> OpBundles;
- CI.getOperandBundlesAsDefs(OpBundles);
- CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
-
- if (isa<FPMathOperator>(V))
- V->copyFastMathFlags(&CI);
-
- State.set(Def, V, Part);
- State.addMetadata(V, &CI);
- }
-}
-
void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
// We should not collect Scalars more than once per VF. Right now, this
// function is called from collectUniformsAndScalars(), which already does
@@ -9316,11 +9259,6 @@ void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
-void VPWidenCallRecipe::execute(VPTransformState &State) {
- State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this,
- *this, State, VectorIntrinsicID);
-}
-
void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Int or FP induction being replicated.");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 68ee6828a3cde..2b4938c02df86 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -434,6 +434,64 @@ void VPInstruction::setFastMathFlags(FastMathFlags FMFNew) {
FMF = FMFNew;
}
+void VPWidenCallRecipe::execute(VPTransformState &State) {
+ auto &CI = *cast<CallInst>(getUnderlyingInstr());
+ assert(!isa<DbgInfoIntrinsic>(CI) &&
+ "DbgInfoIntrinsic should have been dropped during VPlan construction");
+ State.setDebugLocFromInst(&CI);
+
+ SmallVector<Type *, 4> Tys;
+ for (Value *ArgOperand : CI.args())
+ Tys.push_back(
+ ToVectorTy(ArgOperand->getType(), State.VF.getKnownMinValue()));
+
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
+ SmallVector<Type *, 2> TysForDecl = {CI.getType()};
+ SmallVector<Value *, 4> Args;
+ for (const auto &I : enumerate(operands())) {
+ // Some intrinsics have a scalar argument - don't replace it with a
+ // vector.
+ Value *Arg;
+ if (VectorIntrinsicID == Intrinsic::not_intrinsic ||
+ !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
+ Arg = State.get(I.value(), Part);
+ else
+ Arg = State.get(I.value(), VPIteration(0, 0));
+ if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
+ TysForDecl.push_back(Arg->getType());
+ Args.push_back(Arg);
+ }
+
+ Function *VectorF;
+ if (VectorIntrinsicID != Intrinsic::not_intrinsic) {
+ // Use vector version of the intrinsic.
+ if (State.VF.isVector())
+ TysForDecl[0] =
+ VectorType::get(CI.getType()->getScalarType(), State.VF);
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
+ assert(VectorF && "Can't retrieve vector intrinsic.");
+ } else {
+ // Use vector version of the function call.
+ const VFShape Shape = VFShape::get(CI, State.VF, false /*HasGlobalPred*/);
+#ifndef NDEBUG
+ assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
+ "Can't create vector function.");
+#endif
+ VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
+ }
+ SmallVector<OperandBundleDef, 1> OpBundles;
+ CI.getOperandBundlesAsDefs(OpBundles);
+ CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+
+ if (isa<FPMathOperator>(V))
+ V->copyFastMathFlags(&CI);
+
+ State.set(this, V, Part);
+ State.addMetadata(V, &CI);
+ }
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
More information about the llvm-commits
mailing list