[llvm] 1fa6c99 - [VPlan] Move EVL memory recipes to VPlanRecipes.cpp (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 22 10:31:32 PDT 2024
Author: Florian Hahn
Date: 2024-08-22T18:30:49+01:00
New Revision: 1fa6c99a09ccca7558cb3c46fa5d4cbfb4d4bea5
URL: https://github.com/llvm/llvm-project/commit/1fa6c99a09ccca7558cb3c46fa5d4cbfb4d4bea5
DIFF: https://github.com/llvm/llvm-project/commit/1fa6c99a09ccca7558cb3c46fa5d4cbfb4d4bea5.diff
LOG: [VPlan] Move EVL memory recipes to VPlanRecipes.cpp (NFC)
Move VPWiden[Load|Store]EVLRecipe::executeto VPlanRecipes.cpp in line
with other ::execute implementations that don't depend on anything
defined in LoopVectorization.cpp
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2145bb8c9ca872..23d0f39ad93ebe 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9427,102 +9427,6 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
State.ILV->scalarizeInstruction(UI, this, VPIteration(Part, Lane), State);
}
-/// Use all-true mask for reverse rather than actual mask, as it avoids a
-/// dependence w/o affecting the result.
-static Instruction *createReverseEVL(IRBuilderBase &Builder, Value *Operand,
- Value *EVL, const Twine &Name) {
- VectorType *ValTy = cast<VectorType>(Operand->getType());
- Value *AllTrueMask =
- Builder.CreateVectorSplat(ValTy->getElementCount(), Builder.getTrue());
- return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,
- {Operand, AllTrueMask, EVL}, nullptr, Name);
-}
-
-void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
- assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with "
- "explicit vector length.");
- auto *LI = cast<LoadInst>(&Ingredient);
-
- Type *ScalarDataTy = getLoadStoreType(&Ingredient);
- auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
- const Align Alignment = getLoadStoreAlignment(&Ingredient);
- bool CreateGather = !isConsecutive();
-
- auto &Builder = State.Builder;
- State.setDebugLocFrom(getDebugLoc());
- CallInst *NewLI;
- Value *EVL = State.get(getEVL(), VPIteration(0, 0));
- Value *Addr = State.get(getAddr(), 0, !CreateGather);
- Value *Mask = nullptr;
- if (VPValue *VPMask = getMask()) {
- Mask = State.get(VPMask, 0);
- if (isReverse())
- Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
- } else {
- Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
- }
-
- if (CreateGather) {
- NewLI =
- Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},
- nullptr, "wide.masked.gather");
- } else {
- VectorBuilder VBuilder(Builder);
- VBuilder.setEVL(EVL).setMask(Mask);
- NewLI = cast<CallInst>(VBuilder.createVectorInstruction(
- Instruction::Load, DataTy, Addr, "vp.op.load"));
- }
- NewLI->addParamAttr(
- 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
- State.addMetadata(NewLI, LI);
- Instruction *Res = NewLI;
- if (isReverse())
- Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
- State.set(this, Res, 0);
-}
-
-void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
- assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with "
- "explicit vector length.");
- auto *SI = cast<StoreInst>(&Ingredient);
-
- VPValue *StoredValue = getStoredValue();
- bool CreateScatter = !isConsecutive();
- const Align Alignment = getLoadStoreAlignment(&Ingredient);
-
- auto &Builder = State.Builder;
- State.setDebugLocFrom(getDebugLoc());
-
- CallInst *NewSI = nullptr;
- Value *StoredVal = State.get(StoredValue, 0);
- Value *EVL = State.get(getEVL(), VPIteration(0, 0));
- if (isReverse())
- StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");
- Value *Mask = nullptr;
- if (VPValue *VPMask = getMask()) {
- Mask = State.get(VPMask, 0);
- if (isReverse())
- Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
- } else {
- Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
- }
- Value *Addr = State.get(getAddr(), 0, !CreateScatter);
- if (CreateScatter) {
- NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
- Intrinsic::vp_scatter,
- {StoredVal, Addr, Mask, EVL});
- } else {
- VectorBuilder VBuilder(Builder);
- VBuilder.setEVL(EVL).setMask(Mask);
- NewSI = cast<CallInst>(VBuilder.createVectorInstruction(
- Instruction::Store, Type::getVoidTy(EVL->getContext()),
- {StoredVal, Addr}));
- }
- NewSI->addParamAttr(
- 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
- State.addMetadata(NewSI, SI);
-}
-
// Determine how to lower the scalar epilogue, which depends on 1) optimising
// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
// predication, and 4) a TTI hook that analyses whether the loop is suitable
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 63e0e8a8981373..fe1325f4163004 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -2127,7 +2127,63 @@ void VPWidenLoadRecipe::print(raw_ostream &O, const Twine &Indent,
O << " = load ";
printOperands(O, SlotTracker);
}
+#endif
+
+/// Use all-true mask for reverse rather than actual mask, as it avoids a
+/// dependence w/o affecting the result.
+static Instruction *createReverseEVL(IRBuilderBase &Builder, Value *Operand,
+ Value *EVL, const Twine &Name) {
+ VectorType *ValTy = cast<VectorType>(Operand->getType());
+ Value *AllTrueMask =
+ Builder.CreateVectorSplat(ValTy->getElementCount(), Builder.getTrue());
+ return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,
+ {Operand, AllTrueMask, EVL}, nullptr, Name);
+}
+
+void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
+ assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with "
+ "explicit vector length.");
+ auto *LI = cast<LoadInst>(&Ingredient);
+
+ Type *ScalarDataTy = getLoadStoreType(&Ingredient);
+ auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
+ const Align Alignment = getLoadStoreAlignment(&Ingredient);
+ bool CreateGather = !isConsecutive();
+
+ auto &Builder = State.Builder;
+ State.setDebugLocFrom(getDebugLoc());
+ CallInst *NewLI;
+ Value *EVL = State.get(getEVL(), VPIteration(0, 0));
+ Value *Addr = State.get(getAddr(), 0, !CreateGather);
+ Value *Mask = nullptr;
+ if (VPValue *VPMask = getMask()) {
+ Mask = State.get(VPMask, 0);
+ if (isReverse())
+ Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
+ } else {
+ Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
+ }
+
+ if (CreateGather) {
+ NewLI =
+ Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},
+ nullptr, "wide.masked.gather");
+ } else {
+ VectorBuilder VBuilder(Builder);
+ VBuilder.setEVL(EVL).setMask(Mask);
+ NewLI = cast<CallInst>(VBuilder.createVectorInstruction(
+ Instruction::Load, DataTy, Addr, "vp.op.load"));
+ }
+ NewLI->addParamAttr(
+ 0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
+ State.addMetadata(NewLI, LI);
+ Instruction *Res = NewLI;
+ if (isReverse())
+ Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
+ State.set(this, Res, 0);
+}
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenLoadEVLRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN ";
@@ -2183,7 +2239,51 @@ void VPWidenStoreRecipe::print(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN store ";
printOperands(O, SlotTracker);
}
+#endif
+
+void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
+ assert(State.UF == 1 && "Expected only UF == 1 when vectorizing with "
+ "explicit vector length.");
+ auto *SI = cast<StoreInst>(&Ingredient);
+
+ VPValue *StoredValue = getStoredValue();
+ bool CreateScatter = !isConsecutive();
+ const Align Alignment = getLoadStoreAlignment(&Ingredient);
+
+ auto &Builder = State.Builder;
+ State.setDebugLocFrom(getDebugLoc());
+
+ CallInst *NewSI = nullptr;
+ Value *StoredVal = State.get(StoredValue, 0);
+ Value *EVL = State.get(getEVL(), VPIteration(0, 0));
+ if (isReverse())
+ StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");
+ Value *Mask = nullptr;
+ if (VPValue *VPMask = getMask()) {
+ Mask = State.get(VPMask, 0);
+ if (isReverse())
+ Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
+ } else {
+ Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
+ }
+ Value *Addr = State.get(getAddr(), 0, !CreateScatter);
+ if (CreateScatter) {
+ NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
+ Intrinsic::vp_scatter,
+ {StoredVal, Addr, Mask, EVL});
+ } else {
+ VectorBuilder VBuilder(Builder);
+ VBuilder.setEVL(EVL).setMask(Mask);
+ NewSI = cast<CallInst>(VBuilder.createVectorInstruction(
+ Instruction::Store, Type::getVoidTy(EVL->getContext()),
+ {StoredVal, Addr}));
+ }
+ NewSI->addParamAttr(
+ 1, Attribute::getWithAlignment(NewSI->getContext(), Alignment));
+ State.addMetadata(NewSI, SI);
+}
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenStoreEVLRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN vp.store ";
More information about the llvm-commits
mailing list