[llvm] e846778 - [VPlan] Make CallInst optional for VPWidenCallRecipe (NFCI).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed May 1 12:48:35 PDT 2024
Author: Florian Hahn
Date: 2024-05-01T20:48:22+01:00
New Revision: e846778e52f8586b5092c2fd4cdbec2334e31770
URL: https://github.com/llvm/llvm-project/commit/e846778e52f8586b5092c2fd4cdbec2334e31770
DIFF: https://github.com/llvm/llvm-project/commit/e846778e52f8586b5092c2fd4cdbec2334e31770.diff
LOG: [VPlan] Make CallInst optional for VPWidenCallRecipe (NFCI).
Replace relying on the underling CallInst for looking up the called
function and its types by instead adding the called function as operand,
in line with how called functions are handled in CallInst.
Operand bundles, metadata and fast-math flags are optionally used if
there's an underlying CallInst.
This enables creating VPWidenCallRecipes without requiring an underlying
IR instruction.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c44d90f0998eb9..d1c54b928f9fa2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8272,6 +8272,7 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
return nullptr;
SmallVector<VPValue *, 4> Ops(Operands.take_front(CI->arg_size()));
+ Ops.push_back(Operands.back());
// Is it beneficial to perform intrinsic call compared to lib call?
bool ShouldUseVectorIntrinsic =
@@ -8282,7 +8283,7 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
},
Range);
if (ShouldUseVectorIntrinsic)
- return new VPWidenCallRecipe(*CI, make_range(Ops.begin(), Ops.end()), ID,
+ return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), ID,
CI->getDebugLoc());
Function *Variant = nullptr;
@@ -8335,7 +8336,7 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
Ops.insert(Ops.begin() + *MaskPos, Mask);
}
- return new VPWidenCallRecipe(*CI, make_range(Ops.begin(), Ops.end()),
+ return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()),
Intrinsic::not_intrinsic, CI->getDebugLoc(),
Variant);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 71387bf5b7e929..71594be2b965aa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1458,18 +1458,21 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
public:
template <typename IterT>
- VPWidenCallRecipe(CallInst &I, iterator_range<IterT> CallArguments,
+ VPWidenCallRecipe(Value *UV, iterator_range<IterT> CallArguments,
Intrinsic::ID VectorIntrinsicID, DebugLoc DL = {},
Function *Variant = nullptr)
- : VPSingleDefRecipe(VPDef::VPWidenCallSC, CallArguments, &I, DL),
- VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {}
+ : VPSingleDefRecipe(VPDef::VPWidenCallSC, CallArguments, UV, DL),
+ VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {
+ assert(
+ isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
+ "last operand must be the called function");
+ }
~VPWidenCallRecipe() override = default;
VPWidenCallRecipe *clone() override {
- return new VPWidenCallRecipe(*cast<CallInst>(getUnderlyingInstr()),
- operands(), VectorIntrinsicID, getDebugLoc(),
- Variant);
+ return new VPWidenCallRecipe(getUnderlyingValue(), operands(),
+ VectorIntrinsicID, getDebugLoc(), Variant);
}
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
@@ -1477,6 +1480,17 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
/// Produce a widened version of the call instruction.
void execute(VPTransformState &State) override;
+ Function *getCalledScalarFunction() const {
+ return cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
+ }
+
+ operand_range arg_operands() {
+ return make_range(op_begin(), op_begin() + getNumOperands() - 1);
+ }
+ const_operand_range arg_operands() const {
+ return make_range(op_begin(), op_begin() + getNumOperands() - 1);
+ }
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9ec422ec002c82..29ed001ccd2c77 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -51,9 +51,12 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPWidenStoreSC:
return true;
case VPReplicateSC:
- case VPWidenCallSC:
return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
->mayWriteToMemory();
+ case VPWidenCallSC:
+ return !cast<VPWidenCallRecipe>(this)
+ ->getCalledScalarFunction()
+ ->onlyReadsMemory();
case VPBranchOnMaskSC:
case VPScalarIVStepsSC:
case VPPredInstPHISC:
@@ -87,9 +90,12 @@ bool VPRecipeBase::mayReadFromMemory() const {
case VPWidenLoadSC:
return true;
case VPReplicateSC:
- case VPWidenCallSC:
return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
->mayReadFromMemory();
+ case VPWidenCallSC:
+ return !cast<VPWidenCallRecipe>(this)
+ ->getCalledScalarFunction()
+ ->onlyWritesMemory();
case VPBranchOnMaskSC:
case VPPredInstPHISC:
case VPScalarIVStepsSC:
@@ -136,9 +142,10 @@ bool VPRecipeBase::mayHaveSideEffects() const {
default:
return true;
}
- case VPWidenCallSC:
- return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
- ->mayHaveSideEffects();
+ case VPWidenCallSC: {
+ Function *Fn = cast<VPWidenCallRecipe>(this)->getCalledScalarFunction();
+ return mayWriteToMemory() || !Fn->doesNotThrow() || !Fn->willReturn();
+ }
case VPBlendSC:
case VPReductionSC:
case VPScalarIVStepsSC:
@@ -702,8 +709,8 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
void VPWidenCallRecipe::execute(VPTransformState &State) {
assert(State.VF.isVector() && "not widening");
- auto &CI = *cast<CallInst>(getUnderlyingInstr());
- assert(!isa<DbgInfoIntrinsic>(CI) &&
+ Function *CalledScalarFn = getCalledScalarFunction();
+ assert(!isDbgInfoIntrinsic(CalledScalarFn->getIntrinsicID()) &&
"DbgInfoIntrinsic should have been dropped during VPlan construction");
State.setDebugLocFrom(getDebugLoc());
@@ -716,10 +723,10 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
// Add return type if intrinsic is overloaded on it.
if (UseIntrinsic &&
isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1))
- TysForDecl.push_back(
- VectorType::get(CI.getType()->getScalarType(), State.VF));
+ TysForDecl.push_back(VectorType::get(
+ CalledScalarFn->getReturnType()->getScalarType(), State.VF));
SmallVector<Value *, 4> Args;
- for (const auto &I : enumerate(operands())) {
+ for (const auto &I : enumerate(arg_operands())) {
// Some intrinsics have a scalar argument - don't replace it with a
// vector.
Value *Arg;
@@ -752,16 +759,19 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
VectorF = Variant;
}
+ auto *CI = cast_or_null<CallInst>(getUnderlyingInstr());
SmallVector<OperandBundleDef, 1> OpBundles;
- CI.getOperandBundlesAsDefs(OpBundles);
+ if (CI)
+ CI->getOperandBundlesAsDefs(OpBundles);
+
CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
if (isa<FPMathOperator>(V))
- V->copyFastMathFlags(&CI);
+ V->copyFastMathFlags(CI);
if (!V->getType()->isVoidTy())
State.set(this, V, Part);
- State.addMetadata(V, &CI);
+ State.addMetadata(V, CI);
}
}
@@ -770,16 +780,18 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN-CALL ";
- auto *CI = cast<CallInst>(getUnderlyingInstr());
- if (CI->getType()->isVoidTy())
+ Function *CalledFn = getCalledScalarFunction();
+ if (CalledFn->getReturnType()->isVoidTy())
O << "void ";
else {
printAsOperand(O, SlotTracker);
O << " = ";
}
- O << "call @" << CI->getCalledFunction()->getName() << "(";
- printOperands(O, SlotTracker);
+ O << "call @" << CalledFn->getName() << "(";
+ interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) {
+ Op->printAsOperand(O, SlotTracker);
+ });
O << ")";
if (VectorIntrinsicID)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a7337f7aa94d4d..017b00c042f4a4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -75,8 +75,8 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands());
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
NewRecipe = new VPWidenCallRecipe(
- *CI, drop_end(Ingredient.operands()),
- getVectorIntrinsicIDForCall(CI, &TLI), CI->getDebugLoc());
+ CI, Ingredient.operands(), getVectorIntrinsicIDForCall(CI, &TLI),
+ CI->getDebugLoc());
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands());
} else if (auto *CI = dyn_cast<CastInst>(Inst)) {
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 64e9c06db3fe8b..eda4723f67b23d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -895,13 +895,16 @@ TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) {
IntegerType *Int32 = IntegerType::get(C, 32);
FunctionType *FTy = FunctionType::get(Int32, false);
- auto *Call = CallInst::Create(FTy, UndefValue::get(FTy));
+ Function *Fn = Function::Create(FTy, GlobalValue::ExternalLinkage, 0);
+ auto *Call = CallInst::Create(FTy, Fn);
VPValue Op1;
VPValue Op2;
+ VPValue CalledFn(Call->getCalledFunction());
SmallVector<VPValue *, 2> Args;
Args.push_back(&Op1);
Args.push_back(&Op2);
- VPWidenCallRecipe Recipe(*Call, make_range(Args.begin(), Args.end()), false);
+ Args.push_back(&CalledFn);
+ VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false);
EXPECT_TRUE(isa<VPUser>(&Recipe));
VPRecipeBase *BaseR = &Recipe;
EXPECT_TRUE(isa<VPUser>(BaseR));
@@ -912,6 +915,7 @@ TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) {
EXPECT_EQ(&Recipe, VPV->getDefiningRecipe());
delete Call;
+ delete Fn;
}
TEST(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) {
@@ -1157,19 +1161,22 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
{
FunctionType *FTy = FunctionType::get(Int32, false);
- auto *Call = CallInst::Create(FTy, UndefValue::get(FTy));
+ Function *Fn = Function::Create(FTy, GlobalValue::ExternalLinkage, 0);
+ auto *Call = CallInst::Create(FTy, Fn);
VPValue Op1;
VPValue Op2;
- SmallVector<VPValue *, 2> Args;
+ VPValue CalledFn(Call->getCalledFunction());
+ SmallVector<VPValue *, 3> Args;
Args.push_back(&Op1);
Args.push_back(&Op2);
- VPWidenCallRecipe Recipe(*Call, make_range(Args.begin(), Args.end()),
- false);
+ Args.push_back(&CalledFn);
+ VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false);
EXPECT_TRUE(Recipe.mayHaveSideEffects());
EXPECT_TRUE(Recipe.mayReadFromMemory());
EXPECT_TRUE(Recipe.mayWriteToMemory());
EXPECT_TRUE(Recipe.mayReadOrWriteMemory());
delete Call;
+ delete Fn;
}
{
@@ -1181,11 +1188,12 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
auto *Call = CallInst::Create(TheFn->getFunctionType(), TheFn);
VPValue Op1;
VPValue Op2;
- SmallVector<VPValue *, 2> Args;
+ VPValue CalledFn(TheFn);
+ SmallVector<VPValue *, 3> Args;
Args.push_back(&Op1);
Args.push_back(&Op2);
- VPWidenCallRecipe Recipe(*Call, make_range(Args.begin(), Args.end()),
- false);
+ Args.push_back(&CalledFn);
+ VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false);
EXPECT_FALSE(Recipe.mayHaveSideEffects());
EXPECT_FALSE(Recipe.mayReadFromMemory());
EXPECT_FALSE(Recipe.mayWriteToMemory());
More information about the llvm-commits
mailing list