[llvm] [SLP][NFC] Redesign schedule bundle, separate from schedule data, NFC (PR #131625)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 17 08:13:55 PDT 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
That's the initial patch, intended to support revectorization of the
previously vectorized scalars. If the scalar is marked for the
vectorization, it becomes a part of the schedule bundle, used to check
dependencies and then schedule tree entry scalars into a single batch of
instructions. Unfortunately, currently this info is part of the
ScheduleData struct and it does not allow making scalars part of many
bundles. The patch separates schedule bundles from the ScheduleData,
introduces explicit class ScheduleBundle for bundles, allowing later to
extend it to support revectorization of the previously vectorized
scalars.
---
Patch is 73.90 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/131625.diff
1 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+676-469)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d450336cbc3ce..6766b68841a9a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1368,7 +1368,9 @@ namespace slpvectorizer {
/// Bottom Up SLP Vectorizer.
class BoUpSLP {
struct TreeEntry;
- struct ScheduleData;
+ class ScheduleEntity;
+ class ScheduleData;
+ class ScheduleBundle;
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
@@ -1433,7 +1435,8 @@ class BoUpSLP {
/// \returns the vectorization cost of the subtree that starts at \p VL.
/// A negative number means that this is profitable.
- InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {});
+ InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {},
+ InstructionCost ReductionCost = TTI::TCC_Free);
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
@@ -3670,8 +3673,7 @@ class BoUpSLP {
#endif
/// Create a new VectorizableTree entry.
- TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
- std::optional<ScheduleData *> Bundle,
+ TreeEntry *newTreeEntry(ArrayRef<Value *> VL, ScheduleBundle &Bundle,
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
@@ -3688,8 +3690,7 @@ class BoUpSLP {
TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
TreeEntry::EntryState EntryState,
- std::optional<ScheduleData *> Bundle,
- const InstructionsState &S,
+ ScheduleBundle &Bundle, const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
ArrayRef<unsigned> ReorderIndices = {}) {
@@ -3781,22 +3782,23 @@ class BoUpSLP {
}
}
// Update the scheduler bundle to point to this TreeEntry.
- ScheduleData *BundleMember = *Bundle;
- assert((BundleMember || isa<PHINode>(S.getMainOp()) ||
+ assert((!Bundle.getBundle().empty() || isa<PHINode>(S.getMainOp()) ||
isVectorLikeInstWithConstOps(S.getMainOp()) ||
doesNotNeedToSchedule(VL)) &&
"Bundle and VL out of sync");
- if (BundleMember) {
+ if (!Bundle.getBundle().empty()) {
+ auto *BundleMember = Bundle.getBundle().begin();
for (Value *V : VL) {
if (doesNotNeedToBeScheduled(V))
continue;
- if (!BundleMember)
+ if (BundleMember == Bundle.getBundle().end())
continue;
- BundleMember->TE = Last;
- BundleMember = BundleMember->NextInBundle;
+ ++BundleMember;
}
+ assert(BundleMember == Bundle.getBundle().end() &&
+ "Bundle and VL out of sync");
+ Bundle.setTreeEntry(Last);
}
- assert(!BundleMember && "Bundle and VL out of sync");
} else {
// Build a map for gathered scalars to the nodes where they are used.
bool AllConstsOrCasts = true;
@@ -3946,16 +3948,17 @@ class BoUpSLP {
/// is invariant in the calling loop.
bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
Instruction *Inst2) {
- if (!Loc1.Ptr || !isSimple(Inst1) || !isSimple(Inst2))
+ assert(Loc1.Ptr && isSimple(Inst1) && "Expected simple first instrucgion.");
+ if (!isSimple(Inst2))
return true;
// First check if the result is already in the cache.
AliasCacheKey Key = std::make_pair(Inst1, Inst2);
- auto It = AliasCache.find(Key);
- if (It != AliasCache.end())
- return It->second;
+ auto Res = AliasCache.try_emplace(Key);
+ if (!Res.second)
+ return Res.first->second;
bool Aliased = isModOrRefSet(BatchAA.getModRefInfo(Inst2, Loc1));
// Store the result in the cache.
- AliasCache.try_emplace(Key, Aliased);
+ Res.first->getSecond() = Aliased;
AliasCache.try_emplace(std::make_pair(Inst2, Inst1), Aliased);
return Aliased;
}
@@ -3964,7 +3967,7 @@ class BoUpSLP {
/// Cache for alias results.
/// TODO: consider moving this to the AliasAnalysis itself.
- DenseMap<AliasCacheKey, bool> AliasCache;
+ SmallDenseMap<AliasCacheKey, bool> AliasCache;
// Cache for pointerMayBeCaptured calls inside AA. This is preserved
// globally through SLP because we don't perform any action which
@@ -4011,26 +4014,55 @@ class BoUpSLP {
/// List of hashes of vector of loads, which are known to be non vectorizable.
DenseSet<size_t> ListOfKnonwnNonVectorizableLoads;
+ class ScheduleEntity {
+ friend class ScheduleBundle;
+ friend class ScheduleData;
+
+ protected:
+ enum class Kind { ScheduleData, ScheduleBundle };
+ Kind getKind() const { return K; }
+ ScheduleEntity(Kind K) : K(K) {}
+
+ private:
+ /// Used for getting a "good" final ordering of instructions.
+ int SchedulingPriority = 0;
+ /// The kind of the ScheduleEntity.
+ Kind K = Kind::ScheduleData;
+
+ public:
+ ScheduleEntity() = delete;
+ /// Gets/sets the scheduling priority.
+ void setSchedulingPriority(int Priority) { SchedulingPriority = Priority; }
+ int getSchedulingPriority() const { return SchedulingPriority; }
+ bool isReady() const {
+ if (auto *SD = dyn_cast<ScheduleData>(this))
+ return SD->isReady();
+ return cast<ScheduleBundle>(this)->isReady();
+ }
+ static bool classof(const ScheduleEntity *) { return true; }
+ };
+
/// Contains all scheduling relevant data for an instruction.
/// A ScheduleData either represents a single instruction or a member of an
/// instruction bundle (= a group of instructions which is combined into a
/// vector instruction).
- struct ScheduleData {
+ class ScheduleData final : public ScheduleEntity {
+ public:
// The initial value for the dependency counters. It means that the
// dependencies are not calculated yet.
enum { InvalidDeps = -1 };
- ScheduleData() = default;
+ ScheduleData() : ScheduleEntity(Kind::ScheduleData) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleData;
+ }
void init(int BlockSchedulingRegionID, Instruction *I) {
- FirstInBundle = this;
- NextInBundle = nullptr;
NextLoadStore = nullptr;
IsScheduled = false;
SchedulingRegionID = BlockSchedulingRegionID;
clearDependencies();
Inst = I;
- TE = nullptr;
}
/// Verify basic self consistency properties
@@ -4042,20 +4074,9 @@ class BoUpSLP {
}
if (IsScheduled) {
- assert(isSchedulingEntity() &&
- "unexpected scheduled state");
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- assert(BundleMember->hasValidDependencies() &&
- BundleMember->UnscheduledDeps == 0 &&
- "unexpected scheduled state");
- assert((BundleMember == this || !BundleMember->IsScheduled) &&
- "only bundle is marked scheduled");
- }
+ assert(hasValidDependencies() && UnscheduledDeps == 0 &&
+ "unexpected scheduled state");
}
-
- assert(Inst->getParent() == FirstInBundle->Inst->getParent() &&
- "all bundle members must be in same basic block");
}
/// Returns true if the dependency information has been calculated.
@@ -4063,23 +4084,9 @@ class BoUpSLP {
/// a single bundle.
bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
- /// Returns true for single instructions and for bundle representatives
- /// (= the head of a bundle).
- bool isSchedulingEntity() const { return FirstInBundle == this; }
-
- /// Returns true if it represents an instruction bundle and not only a
- /// single instruction.
- bool isPartOfBundle() const {
- return NextInBundle != nullptr || FirstInBundle != this || TE;
- }
-
/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.
- bool isReady() const {
- assert(isSchedulingEntity() &&
- "can't consider non-scheduling entity for ready list");
- return unscheduledDepsInBundle() == 0 && !IsScheduled;
- }
+ bool isReady() const { return UnscheduledDeps == 0 && !IsScheduled; }
/// Modifies the number of unscheduled dependencies for this instruction,
/// and returns the number of remaining dependencies for the containing
@@ -4088,14 +4095,12 @@ class BoUpSLP {
assert(hasValidDependencies() &&
"increment of unscheduled deps would be meaningless");
UnscheduledDeps += Incr;
- return FirstInBundle->unscheduledDepsInBundle();
+ return UnscheduledDeps;
}
/// Sets the number of unscheduled dependencies to the number of
/// dependencies.
- void resetUnscheduledDeps() {
- UnscheduledDeps = Dependencies;
- }
+ void resetUnscheduledDeps() { UnscheduledDeps = Dependencies; }
/// Clears all dependency information.
void clearDependencies() {
@@ -4103,78 +4108,76 @@ class BoUpSLP {
resetUnscheduledDeps();
MemoryDependencies.clear();
ControlDependencies.clear();
+ IsScheduled = false;
}
- int unscheduledDepsInBundle() const {
- assert(isSchedulingEntity() && "only meaningful on the bundle");
- int Sum = 0;
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- if (BundleMember->UnscheduledDeps == InvalidDeps)
- return InvalidDeps;
- Sum += BundleMember->UnscheduledDeps;
- }
- return Sum;
- }
+ /// Gets/sets if the bundle is scheduled.
+ bool isScheduled() const { return IsScheduled; }
+ void setScheduled(bool Scheduled) { IsScheduled = Scheduled; }
- void dump(raw_ostream &OS) const {
- if (isPartOfBundle()) {
- if (!isSchedulingEntity()) {
- OS << "/ " << *Inst << ", part of ";
- FirstInBundle->dump(OS);
- return;
- }
- OS << '[' << *Inst;
- ScheduleData *SD = NextInBundle;
- while (SD) {
- OS << ';' << *SD->Inst;
- SD = SD->NextInBundle;
- }
- OS << ']';
- } else {
- OS << *Inst;
- }
+ /// Gets the number of unscheduled dependencies.
+ int getUnscheduledDeps() const { return UnscheduledDeps; }
+ /// Gets the number of dependencies.
+ int getDependencies() const { return Dependencies; }
+ /// Initializes the number of dependencies.
+ void initDependencies() { Dependencies = 0; }
+ /// Increments the number of dependencies.
+ void incDependencies() { Dependencies++; }
+
+ /// Gets scheduling region ID.
+ int getSchedulingRegionID() const { return SchedulingRegionID; }
+
+ /// Gets the instruction.
+ Instruction *getInst() const { return Inst; }
+
+ /// Gets the list of memory dependencies.
+ ArrayRef<ScheduleData *> getMemoryDependencies() const {
+ return MemoryDependencies;
+ }
+ /// Adds a memory dependency.
+ void addMemoryDependency(ScheduleData *Dep) {
+ MemoryDependencies.push_back(Dep);
+ }
+ /// Gets the list of control dependencies.
+ ArrayRef<ScheduleData *> getControlDependencies() const {
+ return ControlDependencies;
}
+ /// Adds a control dependency.
+ void addControlDependency(ScheduleData *Dep) {
+ ControlDependencies.push_back(Dep);
+ }
+ /// Gets/sets the next load/store instruction in the block.
+ ScheduleData *getNextLoadStore() const { return NextLoadStore; }
+ void setNextLoadStore(ScheduleData *Next) { NextLoadStore = Next; }
+
+ void dump(raw_ostream &OS) const { OS << *Inst; }
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
+ private:
Instruction *Inst = nullptr;
- /// The TreeEntry that this instruction corresponds to.
- TreeEntry *TE = nullptr;
-
- /// Points to the head in an instruction bundle (and always to this for
- /// single instructions).
- ScheduleData *FirstInBundle = nullptr;
-
- /// Single linked list of all instructions in a bundle. Null if it is a
- /// single instruction.
- ScheduleData *NextInBundle = nullptr;
-
/// Single linked list of all memory instructions (e.g. load, store, call)
/// in the block - until the end of the scheduling region.
ScheduleData *NextLoadStore = nullptr;
/// The dependent memory instructions.
/// This list is derived on demand in calculateDependencies().
- SmallVector<ScheduleData *, 4> MemoryDependencies;
+ SmallVector<ScheduleData *> MemoryDependencies;
/// List of instructions which this instruction could be control dependent
/// on. Allowing such nodes to be scheduled below this one could introduce
/// a runtime fault which didn't exist in the original program.
/// ex: this is a load or udiv following a readonly call which inf loops
- SmallVector<ScheduleData *, 4> ControlDependencies;
+ SmallVector<ScheduleData *> ControlDependencies;
/// This ScheduleData is in the current scheduling region if this matches
/// the current SchedulingRegionID of BlockScheduling.
int SchedulingRegionID = 0;
- /// Used for getting a "good" final ordering of instructions.
- int SchedulingPriority = 0;
-
/// The number of dependencies. Constitutes of the number of users of the
/// instruction plus the number of dependent memory instructions (if any).
/// This value is calculated on demand.
@@ -4200,6 +4203,112 @@ class BoUpSLP {
}
#endif
+ class ScheduleBundle final : public ScheduleEntity {
+ /// The schedule data for the instructions in the bundle.
+ SmallVector<ScheduleData *> Bundle;
+ /// True if this bundle is valid.
+ bool IsValid = true;
+ /// The TreeEntry that this instruction corresponds to.
+ TreeEntry *TE = nullptr;
+ ScheduleBundle(bool IsValid)
+ : ScheduleEntity(Kind::ScheduleBundle), IsValid(IsValid) {}
+
+ public:
+ ScheduleBundle() : ScheduleEntity(Kind::ScheduleBundle) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleBundle;
+ }
+
+ /// Verify basic self consistency properties
+ void verify() const {
+ for (const ScheduleData *SD : Bundle) {
+ if (SD->hasValidDependencies()) {
+ assert(SD->getUnscheduledDeps() <= SD->getDependencies() &&
+ "invariant");
+ } else {
+ assert(SD->getUnscheduledDeps() == SD->getDependencies() &&
+ "invariant");
+ }
+
+ if (isScheduled()) {
+ assert(SD->hasValidDependencies() && SD->getUnscheduledDeps() == 0 &&
+ "unexpected scheduled state");
+ }
+ }
+ }
+
+ bool isScheduled() const {
+ return all_of(Bundle,
+ [](const ScheduleData *SD) { return SD->isScheduled(); });
+ }
+
+ int unscheduledDepsInBundle() const {
+ assert(*this && "bundle must not be empty");
+ int Sum = 0;
+ for (const ScheduleData *BundleMember : Bundle) {
+ if (BundleMember->getUnscheduledDeps() == ScheduleData::InvalidDeps)
+ return ScheduleData::InvalidDeps;
+ Sum += BundleMember->getUnscheduledDeps();
+ }
+ return Sum;
+ }
+
+ /// Returns true if the dependency information has been calculated.
+ /// Note that depenendency validity can vary between instructions within
+ /// a single bundle.
+ bool hasValidDependencies() const {
+ return all_of(Bundle, [](const ScheduleData *SD) {
+ return SD->hasValidDependencies();
+ });
+ }
+
+ /// Returns true if it is ready for scheduling, i.e. it has no more
+ /// unscheduled depending instructions/bundles.
+ bool isReady() const {
+ assert(*this && "bundle must not be empty");
+ return unscheduledDepsInBundle() == 0 && !isScheduled();
+ }
+
+ /// Returns the bundle of scheduling data, associated with the current
+ /// instruction.
+ ArrayRef<ScheduleData *> getBundle() { return Bundle; }
+ ArrayRef<const ScheduleData *> getBundle() const { return Bundle; }
+ /// Adds an instruction to the bundle.
+ void add(ScheduleData *SD) { Bundle.push_back(SD); }
+
+ /// Gets/sets the associated tree entry.
+ void setTreeEntry(TreeEntry *TE) { this->TE = TE; }
+ TreeEntry *getTreeEntry() const { return TE; }
+
+ static ScheduleBundle invalid() { return {false}; }
+
+ operator bool() const { return IsValid; }
+
+ void dump(raw_ostream &OS) const {
+ if (!*this) {
+ OS << "[]";
+ return;
+ }
+ OS << '[';
+ interleaveComma(Bundle, OS,
+ [&](const ScheduleData *SD) { OS << *SD->getInst(); });
+ OS << ']';
+ }
+
+ LLVM_DUMP_METHOD void dump() const {
+ dump(dbgs());
+ dbgs() << '\n';
+ }
+ };
+
+#ifndef NDEBUG
+ friend inline raw_ostream &operator<<(raw_ostream &os,
+ const BoUpSLP::ScheduleBundle &Bundle) {
+ Bundle.dump(os);
+ return os;
+ }
+#endif
+
friend struct GraphTraits<BoUpSLP *>;
friend struct DOTGraphTraits<BoUpSLP *>;
@@ -4224,6 +4333,8 @@ class BoUpSLP {
: BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
void clear() {
+ ScheduledBundles.clear();
+ ScheduledBundlesList.clear();
ReadyInsts.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
@@ -4244,6 +4355,8 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Instruction *I) {
+ if (!I)
+ return nullptr;
if (BB != I->getParent())
// Avoid lookup if can't possibly be in map.
return nullptr;
@@ -4254,52 +4367,78 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Value *V) {
- if (auto *I = dyn_cast<Instruction>(V))
- return getScheduleData(I);
- return nullptr;
+ return getScheduleData(dyn_cast<Instruction>(V));
+ }
+
+ ArrayRef<ScheduleBundle *> getScheduleBundles(Value *V) const {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ return {};
+ auto It = ScheduledBundles.find(I);
+ if (It == ScheduledBundles.end())
+ return {};
+ return It->getSecond();
}
bool isInSchedulingRegion(ScheduleData *SD) const {
- return SD->SchedulingRegionID == SchedulingRegionID;
+ return SD->getSchedulingRegionID() == SchedulingRegionID;
+ }
+
+ bool isInSchedulingRegion(const ScheduleBundle &Bundle) const {
+ return all_of(Bundle.getBundle(), [&](const ScheduleData *BundleMember) {
+ return BundleMember->getSchedulingRegionID() == SchedulingRegionID;
+ });
}
/// Marks an instruction as scheduled and puts all dependent ready
/// instructions into the ready-list.
template <typename ReadyListType>
- void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
- SD->IsScheduled = true;
- LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
-
- for (ScheduleData *BundleMember = SD; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
-
+ void schedule(ScheduleEntity *Data, ReadyListType &ReadyList) {
+ auto ProcessBundleMember = [&](ScheduleData *BundleMember,
+ ScheduleBundle *Bundle) {
// Handle the def-use chain dependencies.
// Decrement the unscheduled counter and insert to ready list if ready.
- auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
- ScheduleData *OpDef = getScheduleData(I);
- if (OpDef && OpDef->hasValidDependencies() &&
- OpDef->incrementUnscheduledDeps(-1) == 0) {
+ auto DecrUnsched = [&](ScheduleData *Data, bool IsControl = false) {
+ if ((IsControl || Data->hasValidDependencies()) &&
+ Data->incrementUnscheduledDeps(-1) == 0) {
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/131625
More information about the llvm-commits
mailing list