[llvm] debca7e - [VPlan] Move dropping of poison flags to VPlanTransforms. (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 14 04:29:52 PST 2024
Author: Florian Hahn
Date: 2024-02-14T12:28:58Z
New Revision: debca7ee43522e1702ade36f4954517e35f82886
URL: https://github.com/llvm/llvm-project/commit/debca7ee43522e1702ade36f4954517e35f82886
DIFF: https://github.com/llvm/llvm-project/commit/debca7ee43522e1702ade36f4954517e35f82886.diff
LOG: [VPlan] Move dropping of poison flags to VPlanTransforms. (NFC)
Move collectPoisonGeneratingFlags from InnerLoopVectorizer to
VPlanTransforms and also update its name. collectPoisonGeneratingFlags
already directly drops poison-generating flags, not only collecting it.
This means it is more appropriate to integerate it directly into the
VPlan transform pipeline.
The current implementation still calls back to legal to check if a block
needs predication, which should be improved in the future.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1a7b301c35f2b8..b39bf21cacdbb5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -670,17 +670,6 @@ class InnerLoopVectorizer {
/// running the verifier. Return the preheader of the completed vector loop.
BasicBlock *completeLoopSkeleton();
- /// Collect poison-generating recipes that may generate a poison value that is
- /// used after vectorization, even when their operands are not poison. Those
- /// recipes meet the following conditions:
- /// * Contribute to the address computation of a recipe generating a widen
- /// memory load/store (VPWidenMemoryInstructionRecipe or
- /// VPInterleaveRecipe).
- /// * Such a widen memory load/store has at least one underlying Instruction
- /// that is in a basic block that needs predication and after vectorization
- /// the generated instruction won't be predicated.
- void collectPoisonGeneratingRecipes(VPTransformState &State);
-
/// Allow subclasses to override and print debug traces before/after vplan
/// execution, when trace information is requested.
virtual void printDebugTracesAtStart(){};
@@ -1069,91 +1058,6 @@ static std::string getDebugLocString(const Loop *L) {
}
#endif
-void InnerLoopVectorizer::collectPoisonGeneratingRecipes(
- VPTransformState &State) {
-
- // Collect recipes in the backward slice of `Root` that may generate a poison
- // value that is used after vectorization.
- SmallPtrSet<VPRecipeBase *, 16> Visited;
- auto collectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
- SmallVector<VPRecipeBase *, 16> Worklist;
- Worklist.push_back(Root);
-
- // Traverse the backward slice of Root through its use-def chain.
- while (!Worklist.empty()) {
- VPRecipeBase *CurRec = Worklist.back();
- Worklist.pop_back();
-
- if (!Visited.insert(CurRec).second)
- continue;
-
- // Prune search if we find another recipe generating a widen memory
- // instruction. Widen memory instructions involved in address computation
- // will lead to gather/scatter instructions, which don't need to be
- // handled.
- if (isa<VPWidenMemoryInstructionRecipe>(CurRec) ||
- isa<VPInterleaveRecipe>(CurRec) ||
- isa<VPScalarIVStepsRecipe>(CurRec) ||
- isa<VPCanonicalIVPHIRecipe>(CurRec) ||
- isa<VPActiveLaneMaskPHIRecipe>(CurRec))
- continue;
-
- // This recipe contributes to the address computation of a widen
- // load/store. If the underlying instruction has poison-generating flags,
- // drop them directly.
- if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(CurRec)) {
- RecWithFlags->dropPoisonGeneratingFlags();
- } else {
- Instruction *Instr = dyn_cast_or_null<Instruction>(
- CurRec->getVPSingleValue()->getUnderlyingValue());
- (void)Instr;
- assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
- "found instruction with poison generating flags not covered by "
- "VPRecipeWithIRFlags");
- }
-
- // Add new definitions to the worklist.
- for (VPValue *operand : CurRec->operands())
- if (VPRecipeBase *OpDef = operand->getDefiningRecipe())
- Worklist.push_back(OpDef);
- }
- });
-
- // Traverse all the recipes in the VPlan and collect the poison-generating
- // recipes in the backward slice starting at the address of a VPWidenRecipe or
- // VPInterleaveRecipe.
- auto Iter = vp_depth_first_deep(State.Plan->getEntry());
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
- for (VPRecipeBase &Recipe : *VPBB) {
- if (auto *WidenRec = dyn_cast<VPWidenMemoryInstructionRecipe>(&Recipe)) {
- Instruction &UnderlyingInstr = WidenRec->getIngredient();
- VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
- if (AddrDef && WidenRec->isConsecutive() &&
- Legal->blockNeedsPredication(UnderlyingInstr.getParent()))
- collectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
- } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(&Recipe)) {
- VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
- if (AddrDef) {
- // Check if any member of the interleave group needs predication.
- const InterleaveGroup<Instruction> *InterGroup =
- InterleaveRec->getInterleaveGroup();
- bool NeedPredication = false;
- for (int I = 0, NumMembers = InterGroup->getNumMembers();
- I < NumMembers; ++I) {
- Instruction *Member = InterGroup->getMember(I);
- if (Member)
- NeedPredication |=
- Legal->blockNeedsPredication(Member->getParent());
- }
-
- if (NeedPredication)
- collectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
- }
- }
- }
- }
-}
-
namespace llvm {
// Loop vectorization cost-model hints how the scalar epilogue loop should be
@@ -7591,8 +7495,6 @@ LoopVectorizationPlanner::executePlan(
State.LVer->prepareNoAliasMetadata();
}
- ILV.collectPoisonGeneratingRecipes(State);
-
ILV.printDebugTracesAtStart();
//===------------------------------------------------===//
@@ -8869,6 +8771,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
// in ways that accessing values using original IR values is incorrect.
Plan->disableValue2VPValue();
+ VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) {
+ return Legal->blockNeedsPredication(BB);
+ });
+
// Sink users of fixed-order recurrence past the recipe defining the previous
// value and introduce FirstOrderRecurrenceSplice VPInstructions.
if (!VPlanTransforms::adjustFixedOrderRecurrences(*Plan, Builder))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 71f5285f90236b..16855a9227a186 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1201,3 +1201,86 @@ void VPlanTransforms::addActiveLaneMask(
CompareToReplace->eraseFromParent();
}
}
+
+void VPlanTransforms::dropPoisonGeneratingRecipes(
+ VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication) {
+ // Collect recipes in the backward slice of `Root` that may generate a poison
+ // value that is used after vectorization.
+ SmallPtrSet<VPRecipeBase *, 16> Visited;
+ auto collectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
+ SmallVector<VPRecipeBase *, 16> Worklist;
+ Worklist.push_back(Root);
+
+ // Traverse the backward slice of Root through its use-def chain.
+ while (!Worklist.empty()) {
+ VPRecipeBase *CurRec = Worklist.back();
+ Worklist.pop_back();
+
+ if (!Visited.insert(CurRec).second)
+ continue;
+
+ // Prune search if we find another recipe generating a widen memory
+ // instruction. Widen memory instructions involved in address computation
+ // will lead to gather/scatter instructions, which don't need to be
+ // handled.
+ if (isa<VPWidenMemoryInstructionRecipe>(CurRec) ||
+ isa<VPInterleaveRecipe>(CurRec) ||
+ isa<VPScalarIVStepsRecipe>(CurRec) ||
+ isa<VPCanonicalIVPHIRecipe>(CurRec) ||
+ isa<VPActiveLaneMaskPHIRecipe>(CurRec))
+ continue;
+
+ // This recipe contributes to the address computation of a widen
+ // load/store. If the underlying instruction has poison-generating flags,
+ // drop them directly.
+ if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(CurRec)) {
+ RecWithFlags->dropPoisonGeneratingFlags();
+ } else {
+ Instruction *Instr = dyn_cast_or_null<Instruction>(
+ CurRec->getVPSingleValue()->getUnderlyingValue());
+ (void)Instr;
+ assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
+ "found instruction with poison generating flags not covered by "
+ "VPRecipeWithIRFlags");
+ }
+
+ // Add new definitions to the worklist.
+ for (VPValue *operand : CurRec->operands())
+ if (VPRecipeBase *OpDef = operand->getDefiningRecipe())
+ Worklist.push_back(OpDef);
+ }
+ });
+
+ // Traverse all the recipes in the VPlan and collect the poison-generating
+ // recipes in the backward slice starting at the address of a VPWidenRecipe or
+ // VPInterleaveRecipe.
+ auto Iter = vp_depth_first_deep(Plan.getEntry());
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
+ for (VPRecipeBase &Recipe : *VPBB) {
+ if (auto *WidenRec = dyn_cast<VPWidenMemoryInstructionRecipe>(&Recipe)) {
+ Instruction &UnderlyingInstr = WidenRec->getIngredient();
+ VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
+ if (AddrDef && WidenRec->isConsecutive() &&
+ BlockNeedsPredication(UnderlyingInstr.getParent()))
+ collectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
+ } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(&Recipe)) {
+ VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
+ if (AddrDef) {
+ // Check if any member of the interleave group needs predication.
+ const InterleaveGroup<Instruction> *InterGroup =
+ InterleaveRec->getInterleaveGroup();
+ bool NeedPredication = false;
+ for (int I = 0, NumMembers = InterGroup->getNumMembers();
+ I < NumMembers; ++I) {
+ Instruction *Member = InterGroup->getMember(I);
+ if (Member)
+ NeedPredication |= BlockNeedsPredication(Member->getParent());
+ }
+
+ if (NeedPredication)
+ collectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
+ }
+ }
+ }
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 3bf91115debb7d..4a8e9f5793acfb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -84,6 +84,21 @@ struct VPlanTransforms {
const MapVector<Instruction *, uint64_t> &MinBWs,
LLVMContext &Ctx);
+ /// Drop poison flags from recipes that may generate a poison value that is
+ /// used after vectorization, even when their operands are not poison. Those
+ /// recipes meet the following conditions:
+ /// * Contribute to the address computation of a recipe generating a widen
+ /// memory load/store (VPWidenMemoryInstructionRecipe or
+ /// VPInterleaveRecipe).
+ /// * Such a widen memory load/store has at least one underlying Instruction
+ /// that is in a basic block that needs predication and after vectorization
+ /// the generated instruction won't be predicated.
+ /// Uses \p BlockNeedsPredication to check if a block needs predicating.
+ /// TODO: Replace BlockNeedsPredication callback with retrieving info from
+ /// VPlan directly.
+ static void dropPoisonGeneratingRecipes(
+ VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication);
+
private:
/// Remove redundant VPBasicBlocks by merging them into their predecessor if
/// the predecessor has a single successor.
More information about the llvm-commits
mailing list