[llvm] b807a2b - [VPlan] Move VPReplicateRecipe::execute to VPlanRecipes.cpp (NFC).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun May 4 14:04:40 PDT 2025
Author: Florian Hahn
Date: 2025-05-04T22:02:01+01:00
New Revision: b807a2bc13e96c6819a90e72b99c823c1f3b2d0e
URL: https://github.com/llvm/llvm-project/commit/b807a2bc13e96c6819a90e72b99c823c1f3b2d0e
DIFF: https://github.com/llvm/llvm-project/commit/b807a2bc13e96c6819a90e72b99c823c1f3b2d0e.diff
LOG: [VPlan] Move VPReplicateRecipe::execute to VPlanRecipes.cpp (NFC).
Consolidate ::execute implementation in VPlanRecipes.cpp, in line with
other ::execute implementations.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 13bbfb77e06ce..2682be5866122 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2312,64 +2312,6 @@ static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
return TTI.enableMaskedInterleavedAccessVectorization();
}
-/// A helper function to scalarize a single Instruction in the innermost loop.
-/// Generates a sequence of scalar instances for lane \p Lane. Uses the VPValue
-/// operands from \p RepRecipe instead of \p Instr's operands.
-static void scalarizeInstruction(const Instruction *Instr,
- VPReplicateRecipe *RepRecipe,
- const VPLane &Lane, VPTransformState &State) {
- assert((!Instr->getType()->isAggregateType() ||
- canVectorizeTy(Instr->getType())) &&
- "Expected vectorizable or non-aggregate type.");
-
- // Does this instruction return a value ?
- bool IsVoidRetTy = Instr->getType()->isVoidTy();
-
- Instruction *Cloned = Instr->clone();
- if (!IsVoidRetTy) {
- Cloned->setName(Instr->getName() + ".cloned");
-#if !defined(NDEBUG)
- // Verify that VPlan type inference results agree with the type of the
- // generated values.
- assert(State.TypeAnalysis.inferScalarType(RepRecipe) == Cloned->getType() &&
- "inferred type and type from generated instructions do not match");
-#endif
- }
-
- RepRecipe->applyFlags(*Cloned);
-
- if (auto DL = RepRecipe->getDebugLoc())
- State.setDebugLocFrom(DL);
-
- // Replace the operands of the cloned instructions with their scalar
- // equivalents in the new loop.
- for (const auto &I : enumerate(RepRecipe->operands())) {
- auto InputLane = Lane;
- VPValue *Operand = I.value();
- if (vputils::isUniformAfterVectorization(Operand))
- InputLane = VPLane::getFirstLane();
- Cloned->setOperand(I.index(), State.get(Operand, InputLane));
- }
- State.addNewMetadata(Cloned, Instr);
-
- // Place the cloned scalar in the new loop.
- State.Builder.Insert(Cloned);
-
- State.set(RepRecipe, Cloned, Lane);
-
- // If we just cloned a new assumption, add it the assumption cache.
- if (auto *II = dyn_cast<AssumeInst>(Cloned))
- State.AC->registerAssumption(II);
-
- assert(
- (RepRecipe->getParent()->getParent() ||
- !RepRecipe->getParent()->getPlan()->getVectorLoopRegion() ||
- all_of(RepRecipe->operands(),
- [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); })) &&
- "Expected a recipe is either within a region or all of its operands "
- "are defined outside the vectorized region.");
-}
-
Value *
InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
if (VectorTripCount)
@@ -10081,49 +10023,6 @@ void VPDerivedIVRecipe::execute(VPTransformState &State) {
State.set(this, DerivedIV, VPLane(0));
}
-void VPReplicateRecipe::execute(VPTransformState &State) {
- Instruction *UI = getUnderlyingInstr();
- if (State.Lane) { // Generate a single instance.
- assert((State.VF.isScalar() || !isUniform()) &&
- "uniform recipe shouldn't be predicated");
- assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
- scalarizeInstruction(UI, this, *State.Lane, State);
- // Insert scalar instance packing it into a vector.
- if (State.VF.isVector() && shouldPack()) {
- // If we're constructing lane 0, initialize to start from poison.
- if (State.Lane->isFirstLane()) {
- assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
- Value *Poison = PoisonValue::get(
- VectorType::get(UI->getType(), State.VF));
- State.set(this, Poison);
- }
- State.packScalarIntoVectorizedValue(this, *State.Lane);
- }
- return;
- }
-
- if (IsUniform) {
- // Uniform within VL means we need to generate lane 0.
- scalarizeInstruction(UI, this, VPLane(0), State);
- return;
- }
-
- // A store of a loop varying value to a uniform address only needs the last
- // copy of the store.
- if (isa<StoreInst>(UI) &&
- vputils::isUniformAfterVectorization(getOperand(1))) {
- auto Lane = VPLane::getLastLaneForVF(State.VF);
- scalarizeInstruction(UI, this, VPLane(Lane), State);
- return;
- }
-
- // Generate scalar instances for all VF lanes.
- assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
- const unsigned EndLane = State.VF.getKnownMinValue();
- for (unsigned Lane = 0; Lane < EndLane; ++Lane)
- scalarizeInstruction(UI, this, VPLane(Lane), State);
-}
-
// Determine how to lower the scalar epilogue, which depends on 1) optimising
// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
// predication, and 4) a TTI hook that analyses whether the loop is suitable
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7c01dec80da4e..54e71b04cf9e9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -20,6 +20,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/BasicBlock.h"
@@ -2547,6 +2548,107 @@ void VPReductionEVLRecipe::print(raw_ostream &O, const Twine &Indent,
}
#endif
+/// A helper function to scalarize a single Instruction in the innermost loop.
+/// Generates a sequence of scalar instances for lane \p Lane. Uses the VPValue
+/// operands from \p RepRecipe instead of \p Instr's operands.
+static void scalarizeInstruction(const Instruction *Instr,
+ VPReplicateRecipe *RepRecipe,
+ const VPLane &Lane, VPTransformState &State) {
+ assert((!Instr->getType()->isAggregateType() ||
+ canVectorizeTy(Instr->getType())) &&
+ "Expected vectorizable or non-aggregate type.");
+
+ // Does this instruction return a value ?
+ bool IsVoidRetTy = Instr->getType()->isVoidTy();
+
+ Instruction *Cloned = Instr->clone();
+ if (!IsVoidRetTy) {
+ Cloned->setName(Instr->getName() + ".cloned");
+#if !defined(NDEBUG)
+ // Verify that VPlan type inference results agree with the type of the
+ // generated values.
+ assert(State.TypeAnalysis.inferScalarType(RepRecipe) == Cloned->getType() &&
+ "inferred type and type from generated instructions do not match");
+#endif
+ }
+
+ RepRecipe->applyFlags(*Cloned);
+
+ if (auto DL = RepRecipe->getDebugLoc())
+ State.setDebugLocFrom(DL);
+
+ // Replace the operands of the cloned instructions with their scalar
+ // equivalents in the new loop.
+ for (const auto &I : enumerate(RepRecipe->operands())) {
+ auto InputLane = Lane;
+ VPValue *Operand = I.value();
+ if (vputils::isUniformAfterVectorization(Operand))
+ InputLane = VPLane::getFirstLane();
+ Cloned->setOperand(I.index(), State.get(Operand, InputLane));
+ }
+ State.addNewMetadata(Cloned, Instr);
+
+ // Place the cloned scalar in the new loop.
+ State.Builder.Insert(Cloned);
+
+ State.set(RepRecipe, Cloned, Lane);
+
+ // If we just cloned a new assumption, add it the assumption cache.
+ if (auto *II = dyn_cast<AssumeInst>(Cloned))
+ State.AC->registerAssumption(II);
+
+ assert(
+ (RepRecipe->getParent()->getParent() ||
+ !RepRecipe->getParent()->getPlan()->getVectorLoopRegion() ||
+ all_of(RepRecipe->operands(),
+ [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); })) &&
+ "Expected a recipe is either within a region or all of its operands "
+ "are defined outside the vectorized region.");
+}
+
+void VPReplicateRecipe::execute(VPTransformState &State) {
+ Instruction *UI = getUnderlyingInstr();
+ if (State.Lane) { // Generate a single instance.
+ assert((State.VF.isScalar() || !isUniform()) &&
+ "uniform recipe shouldn't be predicated");
+ assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
+ scalarizeInstruction(UI, this, *State.Lane, State);
+ // Insert scalar instance packing it into a vector.
+ if (State.VF.isVector() && shouldPack()) {
+ // If we're constructing lane 0, initialize to start from poison.
+ if (State.Lane->isFirstLane()) {
+ assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
+ Value *Poison =
+ PoisonValue::get(VectorType::get(UI->getType(), State.VF));
+ State.set(this, Poison);
+ }
+ State.packScalarIntoVectorizedValue(this, *State.Lane);
+ }
+ return;
+ }
+
+ if (IsUniform) {
+ // Uniform within VL means we need to generate lane 0.
+ scalarizeInstruction(UI, this, VPLane(0), State);
+ return;
+ }
+
+ // A store of a loop varying value to a uniform address only needs the last
+ // copy of the store.
+ if (isa<StoreInst>(UI) &&
+ vputils::isUniformAfterVectorization(getOperand(1))) {
+ auto Lane = VPLane::getLastLaneForVF(State.VF);
+ scalarizeInstruction(UI, this, VPLane(Lane), State);
+ return;
+ }
+
+ // Generate scalar instances for all VF lanes.
+ assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
+ const unsigned EndLane = State.VF.getKnownMinValue();
+ for (unsigned Lane = 0; Lane < EndLane; ++Lane)
+ scalarizeInstruction(UI, this, VPLane(Lane), State);
+}
+
bool VPReplicateRecipe::shouldPack() const {
// Find if the recipe is used by a widened recipe via an intervening
// VPPredInstPHIRecipe. In this case, also pack the scalar values in a vector.
More information about the llvm-commits
mailing list