[llvm] [SLP][NFC] Redesign schedule bundle, separate from schedule data, NFC (PR #131625)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 17 08:13:11 PDT 2025
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/131625
That's the initial patch, intended to support revectorization of the
previously vectorized scalars. If the scalar is marked for the
vectorization, it becomes a part of the schedule bundle, used to check
dependencies and then schedule tree entry scalars into a single batch of
instructions. Unfortunately, currently this info is part of the
ScheduleData struct and it does not allow making scalars part of many
bundles. The patch separates schedule bundles from the ScheduleData,
introduces explicit class ScheduleBundle for bundles, allowing later to
extend it to support revectorization of the previously vectorized
scalars.
>From c13a04fcda0e59a9f62ce2043c390017e13d4f7e Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Mon, 17 Mar 2025 15:12:56 +0000
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.5
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 1145 ++++++++++-------
1 file changed, 676 insertions(+), 469 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d450336cbc3ce..6766b68841a9a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1368,7 +1368,9 @@ namespace slpvectorizer {
/// Bottom Up SLP Vectorizer.
class BoUpSLP {
struct TreeEntry;
- struct ScheduleData;
+ class ScheduleEntity;
+ class ScheduleData;
+ class ScheduleBundle;
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
@@ -1433,7 +1435,8 @@ class BoUpSLP {
/// \returns the vectorization cost of the subtree that starts at \p VL.
/// A negative number means that this is profitable.
- InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {});
+ InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = {},
+ InstructionCost ReductionCost = TTI::TCC_Free);
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
@@ -3670,8 +3673,7 @@ class BoUpSLP {
#endif
/// Create a new VectorizableTree entry.
- TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
- std::optional<ScheduleData *> Bundle,
+ TreeEntry *newTreeEntry(ArrayRef<Value *> VL, ScheduleBundle &Bundle,
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
@@ -3688,8 +3690,7 @@ class BoUpSLP {
TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
TreeEntry::EntryState EntryState,
- std::optional<ScheduleData *> Bundle,
- const InstructionsState &S,
+ ScheduleBundle &Bundle, const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
ArrayRef<unsigned> ReorderIndices = {}) {
@@ -3781,22 +3782,23 @@ class BoUpSLP {
}
}
// Update the scheduler bundle to point to this TreeEntry.
- ScheduleData *BundleMember = *Bundle;
- assert((BundleMember || isa<PHINode>(S.getMainOp()) ||
+ assert((!Bundle.getBundle().empty() || isa<PHINode>(S.getMainOp()) ||
isVectorLikeInstWithConstOps(S.getMainOp()) ||
doesNotNeedToSchedule(VL)) &&
"Bundle and VL out of sync");
- if (BundleMember) {
+ if (!Bundle.getBundle().empty()) {
+ auto *BundleMember = Bundle.getBundle().begin();
for (Value *V : VL) {
if (doesNotNeedToBeScheduled(V))
continue;
- if (!BundleMember)
+ if (BundleMember == Bundle.getBundle().end())
continue;
- BundleMember->TE = Last;
- BundleMember = BundleMember->NextInBundle;
+ ++BundleMember;
}
+ assert(BundleMember == Bundle.getBundle().end() &&
+ "Bundle and VL out of sync");
+ Bundle.setTreeEntry(Last);
}
- assert(!BundleMember && "Bundle and VL out of sync");
} else {
// Build a map for gathered scalars to the nodes where they are used.
bool AllConstsOrCasts = true;
@@ -3946,16 +3948,17 @@ class BoUpSLP {
/// is invariant in the calling loop.
bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
Instruction *Inst2) {
- if (!Loc1.Ptr || !isSimple(Inst1) || !isSimple(Inst2))
+ assert(Loc1.Ptr && isSimple(Inst1) && "Expected simple first instrucgion.");
+ if (!isSimple(Inst2))
return true;
// First check if the result is already in the cache.
AliasCacheKey Key = std::make_pair(Inst1, Inst2);
- auto It = AliasCache.find(Key);
- if (It != AliasCache.end())
- return It->second;
+ auto Res = AliasCache.try_emplace(Key);
+ if (!Res.second)
+ return Res.first->second;
bool Aliased = isModOrRefSet(BatchAA.getModRefInfo(Inst2, Loc1));
// Store the result in the cache.
- AliasCache.try_emplace(Key, Aliased);
+ Res.first->getSecond() = Aliased;
AliasCache.try_emplace(std::make_pair(Inst2, Inst1), Aliased);
return Aliased;
}
@@ -3964,7 +3967,7 @@ class BoUpSLP {
/// Cache for alias results.
/// TODO: consider moving this to the AliasAnalysis itself.
- DenseMap<AliasCacheKey, bool> AliasCache;
+ SmallDenseMap<AliasCacheKey, bool> AliasCache;
// Cache for pointerMayBeCaptured calls inside AA. This is preserved
// globally through SLP because we don't perform any action which
@@ -4011,26 +4014,55 @@ class BoUpSLP {
/// List of hashes of vector of loads, which are known to be non vectorizable.
DenseSet<size_t> ListOfKnonwnNonVectorizableLoads;
+ class ScheduleEntity {
+ friend class ScheduleBundle;
+ friend class ScheduleData;
+
+ protected:
+ enum class Kind { ScheduleData, ScheduleBundle };
+ Kind getKind() const { return K; }
+ ScheduleEntity(Kind K) : K(K) {}
+
+ private:
+ /// Used for getting a "good" final ordering of instructions.
+ int SchedulingPriority = 0;
+ /// The kind of the ScheduleEntity.
+ Kind K = Kind::ScheduleData;
+
+ public:
+ ScheduleEntity() = delete;
+ /// Gets/sets the scheduling priority.
+ void setSchedulingPriority(int Priority) { SchedulingPriority = Priority; }
+ int getSchedulingPriority() const { return SchedulingPriority; }
+ bool isReady() const {
+ if (auto *SD = dyn_cast<ScheduleData>(this))
+ return SD->isReady();
+ return cast<ScheduleBundle>(this)->isReady();
+ }
+ static bool classof(const ScheduleEntity *) { return true; }
+ };
+
/// Contains all scheduling relevant data for an instruction.
/// A ScheduleData either represents a single instruction or a member of an
/// instruction bundle (= a group of instructions which is combined into a
/// vector instruction).
- struct ScheduleData {
+ class ScheduleData final : public ScheduleEntity {
+ public:
// The initial value for the dependency counters. It means that the
// dependencies are not calculated yet.
enum { InvalidDeps = -1 };
- ScheduleData() = default;
+ ScheduleData() : ScheduleEntity(Kind::ScheduleData) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleData;
+ }
void init(int BlockSchedulingRegionID, Instruction *I) {
- FirstInBundle = this;
- NextInBundle = nullptr;
NextLoadStore = nullptr;
IsScheduled = false;
SchedulingRegionID = BlockSchedulingRegionID;
clearDependencies();
Inst = I;
- TE = nullptr;
}
/// Verify basic self consistency properties
@@ -4042,20 +4074,9 @@ class BoUpSLP {
}
if (IsScheduled) {
- assert(isSchedulingEntity() &&
- "unexpected scheduled state");
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- assert(BundleMember->hasValidDependencies() &&
- BundleMember->UnscheduledDeps == 0 &&
- "unexpected scheduled state");
- assert((BundleMember == this || !BundleMember->IsScheduled) &&
- "only bundle is marked scheduled");
- }
+ assert(hasValidDependencies() && UnscheduledDeps == 0 &&
+ "unexpected scheduled state");
}
-
- assert(Inst->getParent() == FirstInBundle->Inst->getParent() &&
- "all bundle members must be in same basic block");
}
/// Returns true if the dependency information has been calculated.
@@ -4063,23 +4084,9 @@ class BoUpSLP {
/// a single bundle.
bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
- /// Returns true for single instructions and for bundle representatives
- /// (= the head of a bundle).
- bool isSchedulingEntity() const { return FirstInBundle == this; }
-
- /// Returns true if it represents an instruction bundle and not only a
- /// single instruction.
- bool isPartOfBundle() const {
- return NextInBundle != nullptr || FirstInBundle != this || TE;
- }
-
/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.
- bool isReady() const {
- assert(isSchedulingEntity() &&
- "can't consider non-scheduling entity for ready list");
- return unscheduledDepsInBundle() == 0 && !IsScheduled;
- }
+ bool isReady() const { return UnscheduledDeps == 0 && !IsScheduled; }
/// Modifies the number of unscheduled dependencies for this instruction,
/// and returns the number of remaining dependencies for the containing
@@ -4088,14 +4095,12 @@ class BoUpSLP {
assert(hasValidDependencies() &&
"increment of unscheduled deps would be meaningless");
UnscheduledDeps += Incr;
- return FirstInBundle->unscheduledDepsInBundle();
+ return UnscheduledDeps;
}
/// Sets the number of unscheduled dependencies to the number of
/// dependencies.
- void resetUnscheduledDeps() {
- UnscheduledDeps = Dependencies;
- }
+ void resetUnscheduledDeps() { UnscheduledDeps = Dependencies; }
/// Clears all dependency information.
void clearDependencies() {
@@ -4103,78 +4108,76 @@ class BoUpSLP {
resetUnscheduledDeps();
MemoryDependencies.clear();
ControlDependencies.clear();
+ IsScheduled = false;
}
- int unscheduledDepsInBundle() const {
- assert(isSchedulingEntity() && "only meaningful on the bundle");
- int Sum = 0;
- for (const ScheduleData *BundleMember = this; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- if (BundleMember->UnscheduledDeps == InvalidDeps)
- return InvalidDeps;
- Sum += BundleMember->UnscheduledDeps;
- }
- return Sum;
- }
+ /// Gets/sets if the bundle is scheduled.
+ bool isScheduled() const { return IsScheduled; }
+ void setScheduled(bool Scheduled) { IsScheduled = Scheduled; }
- void dump(raw_ostream &OS) const {
- if (isPartOfBundle()) {
- if (!isSchedulingEntity()) {
- OS << "/ " << *Inst << ", part of ";
- FirstInBundle->dump(OS);
- return;
- }
- OS << '[' << *Inst;
- ScheduleData *SD = NextInBundle;
- while (SD) {
- OS << ';' << *SD->Inst;
- SD = SD->NextInBundle;
- }
- OS << ']';
- } else {
- OS << *Inst;
- }
+ /// Gets the number of unscheduled dependencies.
+ int getUnscheduledDeps() const { return UnscheduledDeps; }
+ /// Gets the number of dependencies.
+ int getDependencies() const { return Dependencies; }
+ /// Initializes the number of dependencies.
+ void initDependencies() { Dependencies = 0; }
+ /// Increments the number of dependencies.
+ void incDependencies() { Dependencies++; }
+
+ /// Gets scheduling region ID.
+ int getSchedulingRegionID() const { return SchedulingRegionID; }
+
+ /// Gets the instruction.
+ Instruction *getInst() const { return Inst; }
+
+ /// Gets the list of memory dependencies.
+ ArrayRef<ScheduleData *> getMemoryDependencies() const {
+ return MemoryDependencies;
+ }
+ /// Adds a memory dependency.
+ void addMemoryDependency(ScheduleData *Dep) {
+ MemoryDependencies.push_back(Dep);
+ }
+ /// Gets the list of control dependencies.
+ ArrayRef<ScheduleData *> getControlDependencies() const {
+ return ControlDependencies;
}
+ /// Adds a control dependency.
+ void addControlDependency(ScheduleData *Dep) {
+ ControlDependencies.push_back(Dep);
+ }
+ /// Gets/sets the next load/store instruction in the block.
+ ScheduleData *getNextLoadStore() const { return NextLoadStore; }
+ void setNextLoadStore(ScheduleData *Next) { NextLoadStore = Next; }
+
+ void dump(raw_ostream &OS) const { OS << *Inst; }
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
+ private:
Instruction *Inst = nullptr;
- /// The TreeEntry that this instruction corresponds to.
- TreeEntry *TE = nullptr;
-
- /// Points to the head in an instruction bundle (and always to this for
- /// single instructions).
- ScheduleData *FirstInBundle = nullptr;
-
- /// Single linked list of all instructions in a bundle. Null if it is a
- /// single instruction.
- ScheduleData *NextInBundle = nullptr;
-
/// Single linked list of all memory instructions (e.g. load, store, call)
/// in the block - until the end of the scheduling region.
ScheduleData *NextLoadStore = nullptr;
/// The dependent memory instructions.
/// This list is derived on demand in calculateDependencies().
- SmallVector<ScheduleData *, 4> MemoryDependencies;
+ SmallVector<ScheduleData *> MemoryDependencies;
/// List of instructions which this instruction could be control dependent
/// on. Allowing such nodes to be scheduled below this one could introduce
/// a runtime fault which didn't exist in the original program.
/// ex: this is a load or udiv following a readonly call which inf loops
- SmallVector<ScheduleData *, 4> ControlDependencies;
+ SmallVector<ScheduleData *> ControlDependencies;
/// This ScheduleData is in the current scheduling region if this matches
/// the current SchedulingRegionID of BlockScheduling.
int SchedulingRegionID = 0;
- /// Used for getting a "good" final ordering of instructions.
- int SchedulingPriority = 0;
-
/// The number of dependencies. Constitutes of the number of users of the
/// instruction plus the number of dependent memory instructions (if any).
/// This value is calculated on demand.
@@ -4200,6 +4203,112 @@ class BoUpSLP {
}
#endif
+ class ScheduleBundle final : public ScheduleEntity {
+ /// The schedule data for the instructions in the bundle.
+ SmallVector<ScheduleData *> Bundle;
+ /// True if this bundle is valid.
+ bool IsValid = true;
+ /// The TreeEntry that this instruction corresponds to.
+ TreeEntry *TE = nullptr;
+ ScheduleBundle(bool IsValid)
+ : ScheduleEntity(Kind::ScheduleBundle), IsValid(IsValid) {}
+
+ public:
+ ScheduleBundle() : ScheduleEntity(Kind::ScheduleBundle) {}
+ static bool classof(const ScheduleEntity *Entity) {
+ return Entity->getKind() == Kind::ScheduleBundle;
+ }
+
+ /// Verify basic self consistency properties
+ void verify() const {
+ for (const ScheduleData *SD : Bundle) {
+ if (SD->hasValidDependencies()) {
+ assert(SD->getUnscheduledDeps() <= SD->getDependencies() &&
+ "invariant");
+ } else {
+ assert(SD->getUnscheduledDeps() == SD->getDependencies() &&
+ "invariant");
+ }
+
+ if (isScheduled()) {
+ assert(SD->hasValidDependencies() && SD->getUnscheduledDeps() == 0 &&
+ "unexpected scheduled state");
+ }
+ }
+ }
+
+ bool isScheduled() const {
+ return all_of(Bundle,
+ [](const ScheduleData *SD) { return SD->isScheduled(); });
+ }
+
+ int unscheduledDepsInBundle() const {
+ assert(*this && "bundle must not be empty");
+ int Sum = 0;
+ for (const ScheduleData *BundleMember : Bundle) {
+ if (BundleMember->getUnscheduledDeps() == ScheduleData::InvalidDeps)
+ return ScheduleData::InvalidDeps;
+ Sum += BundleMember->getUnscheduledDeps();
+ }
+ return Sum;
+ }
+
+ /// Returns true if the dependency information has been calculated.
+ /// Note that depenendency validity can vary between instructions within
+ /// a single bundle.
+ bool hasValidDependencies() const {
+ return all_of(Bundle, [](const ScheduleData *SD) {
+ return SD->hasValidDependencies();
+ });
+ }
+
+ /// Returns true if it is ready for scheduling, i.e. it has no more
+ /// unscheduled depending instructions/bundles.
+ bool isReady() const {
+ assert(*this && "bundle must not be empty");
+ return unscheduledDepsInBundle() == 0 && !isScheduled();
+ }
+
+ /// Returns the bundle of scheduling data, associated with the current
+ /// instruction.
+ ArrayRef<ScheduleData *> getBundle() { return Bundle; }
+ ArrayRef<const ScheduleData *> getBundle() const { return Bundle; }
+ /// Adds an instruction to the bundle.
+ void add(ScheduleData *SD) { Bundle.push_back(SD); }
+
+ /// Gets/sets the associated tree entry.
+ void setTreeEntry(TreeEntry *TE) { this->TE = TE; }
+ TreeEntry *getTreeEntry() const { return TE; }
+
+ static ScheduleBundle invalid() { return {false}; }
+
+ operator bool() const { return IsValid; }
+
+ void dump(raw_ostream &OS) const {
+ if (!*this) {
+ OS << "[]";
+ return;
+ }
+ OS << '[';
+ interleaveComma(Bundle, OS,
+ [&](const ScheduleData *SD) { OS << *SD->getInst(); });
+ OS << ']';
+ }
+
+ LLVM_DUMP_METHOD void dump() const {
+ dump(dbgs());
+ dbgs() << '\n';
+ }
+ };
+
+#ifndef NDEBUG
+ friend inline raw_ostream &operator<<(raw_ostream &os,
+ const BoUpSLP::ScheduleBundle &Bundle) {
+ Bundle.dump(os);
+ return os;
+ }
+#endif
+
friend struct GraphTraits<BoUpSLP *>;
friend struct DOTGraphTraits<BoUpSLP *>;
@@ -4224,6 +4333,8 @@ class BoUpSLP {
: BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
void clear() {
+ ScheduledBundles.clear();
+ ScheduledBundlesList.clear();
ReadyInsts.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
@@ -4244,6 +4355,8 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Instruction *I) {
+ if (!I)
+ return nullptr;
if (BB != I->getParent())
// Avoid lookup if can't possibly be in map.
return nullptr;
@@ -4254,52 +4367,78 @@ class BoUpSLP {
}
ScheduleData *getScheduleData(Value *V) {
- if (auto *I = dyn_cast<Instruction>(V))
- return getScheduleData(I);
- return nullptr;
+ return getScheduleData(dyn_cast<Instruction>(V));
+ }
+
+ ArrayRef<ScheduleBundle *> getScheduleBundles(Value *V) const {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ return {};
+ auto It = ScheduledBundles.find(I);
+ if (It == ScheduledBundles.end())
+ return {};
+ return It->getSecond();
}
bool isInSchedulingRegion(ScheduleData *SD) const {
- return SD->SchedulingRegionID == SchedulingRegionID;
+ return SD->getSchedulingRegionID() == SchedulingRegionID;
+ }
+
+ bool isInSchedulingRegion(const ScheduleBundle &Bundle) const {
+ return all_of(Bundle.getBundle(), [&](const ScheduleData *BundleMember) {
+ return BundleMember->getSchedulingRegionID() == SchedulingRegionID;
+ });
}
/// Marks an instruction as scheduled and puts all dependent ready
/// instructions into the ready-list.
template <typename ReadyListType>
- void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
- SD->IsScheduled = true;
- LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
-
- for (ScheduleData *BundleMember = SD; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
-
+ void schedule(ScheduleEntity *Data, ReadyListType &ReadyList) {
+ auto ProcessBundleMember = [&](ScheduleData *BundleMember,
+ ScheduleBundle *Bundle) {
// Handle the def-use chain dependencies.
// Decrement the unscheduled counter and insert to ready list if ready.
- auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
- ScheduleData *OpDef = getScheduleData(I);
- if (OpDef && OpDef->hasValidDependencies() &&
- OpDef->incrementUnscheduledDeps(-1) == 0) {
+ auto DecrUnsched = [&](ScheduleData *Data, bool IsControl = false) {
+ if ((IsControl || Data->hasValidDependencies()) &&
+ Data->incrementUnscheduledDeps(-1) == 0) {
// There are no more unscheduled dependencies after
// decrementing, so we can put the dependent instruction
// into the ready list.
- ScheduleData *DepBundle = OpDef->FirstInBundle;
- assert(!DepBundle->IsScheduled &&
+ if (ArrayRef<ScheduleBundle *> Bundles =
+ getScheduleBundles(Data->getInst());
+ !Bundles.empty()) {
+ for (ScheduleBundle *Bundle : Bundles) {
+ if (Bundle->unscheduledDepsInBundle() == 0) {
+ assert(!Bundle->isScheduled() &&
+ "already scheduled bundle gets ready");
+ ReadyList.insert(Bundle);
+ LLVM_DEBUG(dbgs()
+ << "SLP: gets ready: " << *Bundle << "\n");
+ }
+ }
+ return;
+ }
+ assert(!Data->isScheduled() &&
"already scheduled bundle gets ready");
- ReadyList.insert(DepBundle);
- LLVM_DEBUG(dbgs()
- << "SLP: gets ready (def): " << *DepBundle << "\n");
+ ReadyList.insert(Data);
+ LLVM_DEBUG(dbgs() << "SLP: gets ready: " << *Data << "\n");
}
};
+ auto DecrUnschedForInst = [&](Instruction *I) {
+ if (ScheduleData *OpSD = getScheduleData(I))
+ DecrUnsched(OpSD, /*IsControl*/false);
+ };
+
// If BundleMember is a vector bundle, its operands may have been
// reordered during buildTree(). We therefore need to get its operands
// through the TreeEntry.
- if (TreeEntry *TE = BundleMember->TE) {
+ if (Bundle) {
// Need to search for the lane since the tree entry can be reordered.
- auto *In = BundleMember->Inst;
- int Lane = std::distance(TE->Scalars.begin(),
- find(TE->Scalars, In));
+ auto *In = BundleMember->getInst();
+ int Lane = std::distance(Bundle->getTreeEntry()->Scalars.begin(),
+ find(Bundle->getTreeEntry()->Scalars, In));
assert(Lane >= 0 && "Lane not set");
// Since vectorization tree is being built recursively this assertion
@@ -4311,46 +4450,58 @@ class BoUpSLP {
assert(
In &&
(isa<ExtractValueInst, ExtractElementInst, IntrinsicInst>(In) ||
- In->getNumOperands() == TE->getNumOperands()) &&
+ In->getNumOperands() ==
+ Bundle->getTreeEntry()->getNumOperands()) &&
"Missed TreeEntry operands?");
- for (unsigned OpIdx : seq<unsigned>(TE->getNumOperands()))
- if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane]))
- DecrUnsched(I);
+ for (unsigned OpIdx :
+ seq<unsigned>(Bundle->getTreeEntry()->getNumOperands()))
+ if (auto *I = dyn_cast<Instruction>(
+ Bundle->getTreeEntry()->getOperand(OpIdx)[Lane])) {
+ LLVM_DEBUG(dbgs()
+ << "SLP: check for readiness (def): " << *I << "\n");
+ DecrUnschedForInst(I);
+ }
} else {
// If BundleMember is a stand-alone instruction, no operand reordering
// has taken place, so we directly access its operands.
- for (Use &U : BundleMember->Inst->operands())
- if (auto *I = dyn_cast<Instruction>(U.get()))
- DecrUnsched(I);
+ for (Use &U : BundleMember->getInst()->operands())
+ if (auto *I = dyn_cast<Instruction>(U.get())) {
+ LLVM_DEBUG(dbgs()
+ << "SLP: check for readiness (def): " << *I << "\n");
+ DecrUnschedForInst(I);
+ }
}
// Handle the memory dependencies.
- for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
- if (MemoryDepSD->hasValidDependencies() &&
- MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
- // There are no more unscheduled dependencies after decrementing,
- // so we can put the dependent instruction into the ready list.
- ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
- assert(!DepBundle->IsScheduled &&
- "already scheduled bundle gets ready");
- ReadyList.insert(DepBundle);
- LLVM_DEBUG(dbgs()
- << "SLP: gets ready (mem): " << *DepBundle << "\n");
- }
+ for (ScheduleData *MemoryDep : BundleMember->getMemoryDependencies()) {
+ // There are no more unscheduled dependencies after decrementing,
+ // so we can put the dependent instruction into the ready list.
+ LLVM_DEBUG(dbgs()
+ << "SLP: check for readiness (mem): " << *MemoryDep
+ << "\n");
+ DecrUnsched(MemoryDep);
}
// Handle the control dependencies.
- for (ScheduleData *DepSD : BundleMember->ControlDependencies) {
- if (DepSD->incrementUnscheduledDeps(-1) == 0) {
- // There are no more unscheduled dependencies after decrementing,
- // so we can put the dependent instruction into the ready list.
- ScheduleData *DepBundle = DepSD->FirstInBundle;
- assert(!DepBundle->IsScheduled &&
- "already scheduled bundle gets ready");
- ReadyList.insert(DepBundle);
- LLVM_DEBUG(dbgs()
- << "SLP: gets ready (ctl): " << *DepBundle << "\n");
- }
+ for (ScheduleData *Dep : BundleMember->getControlDependencies()) {
+ // There are no more unscheduled dependencies after decrementing,
+ // so we can put the dependent instruction into the ready list.
+ LLVM_DEBUG(dbgs()
+ << "SLP: check for readiness (ctrl): " << *Dep << "\n");
+ DecrUnsched(Dep, /*IsControl=*/true);
}
+ };
+ if (auto *SD = dyn_cast<ScheduleData>(Data)) {
+ SD->setScheduled(/*Scheduled=*/true);
+ LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
+ ProcessBundleMember(SD, nullptr);
+ } else {
+ ScheduleBundle &Bundle = *cast<ScheduleBundle>(Data);
+ for_each(Bundle.getBundle(), [](ScheduleData *SD) {
+ SD->setScheduled(/*Scheduled=*/true);
+ });
+ LLVM_DEBUG(dbgs() << "SLP: schedule " << Bundle << "\n");
+ for (ScheduleData *SD : Bundle.getBundle())
+ ProcessBundleMember(SD, &Bundle);
}
}
@@ -4364,30 +4515,49 @@ class BoUpSLP {
"Not a valid scheduling region?");
for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
+ ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(I);
+ if (!Bundles.empty()) {
+ for (ScheduleBundle *Bundle : Bundles) {
+ assert(isInSchedulingRegion(*Bundle) &&
+ "primary schedule data not in window?");
+ Bundle->verify();
+ }
+ continue;
+ }
auto *SD = getScheduleData(I);
if (!SD)
continue;
assert(isInSchedulingRegion(SD) &&
"primary schedule data not in window?");
- assert(isInSchedulingRegion(SD->FirstInBundle) &&
- "entire bundle in window!");
SD->verify();
}
- for (auto *SD : ReadyInsts) {
- assert(SD->isSchedulingEntity() && SD->isReady() &&
- "item in ready list not ready?");
- (void)SD;
+ for (const ScheduleEntity *Bundle : ReadyInsts) {
+ assert(Bundle->isReady() && "item in ready list not ready?");
+ (void)Bundle;
}
}
/// Put all instructions into the ReadyList which are ready for scheduling.
template <typename ReadyListType>
void initialFillReadyList(ReadyListType &ReadyList) {
+ SmallPtrSet<ScheduleBundle *, 16> Visited;
for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
ScheduleData *SD = getScheduleData(I);
- if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies() &&
- SD->isReady()) {
+ if (SD && SD->hasValidDependencies() && SD->isReady()) {
+ if (ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(I);
+ !Bundles.empty()) {
+ for (ScheduleBundle *Bundle : Bundles) {
+ if (!Visited.insert(Bundle).second)
+ continue;
+ if (Bundle->hasValidDependencies() && Bundle->isReady()) {
+ ReadyList.insert(Bundle);
+ LLVM_DEBUG(dbgs() << "SLP: initially in ready list: "
+ << *Bundle << "\n");
+ }
+ }
+ continue;
+ }
ReadyList.insert(SD);
LLVM_DEBUG(dbgs()
<< "SLP: initially in ready list: " << *SD << "\n");
@@ -4397,20 +4567,17 @@ class BoUpSLP {
/// Build a bundle from the ScheduleData nodes corresponding to the
/// scalar instruction for each lane.
- ScheduleData *buildBundle(ArrayRef<Value *> VL);
+ ScheduleBundle &buildBundle(ArrayRef<Value *> VL);
/// Checks if a bundle of instructions can be scheduled, i.e. has no
/// cyclic dependencies. This is only a dry-run, no instructions are
/// actually moved at this stage.
/// \returns the scheduling bundle. The returned Optional value is not
/// std::nullopt if \p VL is allowed to be scheduled.
- std::optional<ScheduleData *>
+ std::optional<ScheduleBundle *>
tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
const InstructionsState &S);
- /// Un-bundles a group of instructions.
- void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
-
/// Allocates schedule data chunk.
ScheduleData *allocateScheduleDataChunks();
@@ -4426,7 +4593,7 @@ class BoUpSLP {
/// Updates the dependency information of a bundle and of all instructions/
/// bundles which depend on the original bundle.
- void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
+ void calculateDependencies(ScheduleBundle &Bundle, bool InsertInReadyList,
BoUpSLP *SLP);
/// Sets all instruction in the scheduling region to un-scheduled.
@@ -4447,10 +4614,16 @@ class BoUpSLP {
/// Attaches ScheduleData to Instruction.
/// Note that the mapping survives during all vectorization iterations, i.e.
/// ScheduleData structures are recycled.
- DenseMap<Instruction *, ScheduleData *> ScheduleDataMap;
+ SmallDenseMap<Instruction *, ScheduleData *> ScheduleDataMap;
+
+ /// Attaches ScheduleBundle to Instruction.
+ SmallDenseMap<Instruction *, SmallVector<ScheduleBundle *>>
+ ScheduledBundles;
+ /// The list of ScheduleBundles.
+ SmallVector<std::unique_ptr<ScheduleBundle>> ScheduledBundlesList;
/// The ready-list for scheduling (only used for the dry-run).
- SetVector<ScheduleData *> ReadyInsts;
+ SetVector<ScheduleEntity *> ReadyInsts;
/// The first instruction of the scheduling region.
Instruction *ScheduleStart = nullptr;
@@ -7836,24 +8009,6 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
GatheredLoadsEntriesFirst.reset();
}
-/// \return true if the specified list of values has only one instruction that
-/// requires scheduling, false otherwise.
-#ifndef NDEBUG
-static bool needToScheduleSingleInstruction(ArrayRef<Value *> VL) {
- Value *NeedsScheduling = nullptr;
- for (Value *V : VL) {
- if (doesNotNeedToBeScheduled(V))
- continue;
- if (!NeedsScheduling) {
- NeedsScheduling = V;
- continue;
- }
- return false;
- }
- return NeedsScheduling;
-}
-#endif
-
/// Generates key/subkey pair for the given value to provide effective sorting
/// of the values and better detection of the vectorizable values sequences. The
/// keys/subkeys can be used for better sorting of the values themselves (keys)
@@ -8675,7 +8830,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
!hasFullVectorsOrPowerOf2(*TTI, VL.front()->getType(), VL.size())) {
LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
"for nodes with padding.\n");
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
return false;
}
LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
@@ -8700,7 +8856,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
// vectorization (div/rem are not allowed).
if (!getSameOpcode(NonUniqueValueVL, *TLI).valid()) {
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
return false;
}
VL = NonUniqueValueVL;
@@ -8708,7 +8865,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
return true;
}
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
return false;
}
VL = UniqueValues;
@@ -8723,7 +8881,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
// place to insert a shuffle if we need to, so just avoid that issue.
if (S && isa<CatchSwitchInst>(S.getMainOp()->getParent()->getTerminator())) {
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
return;
}
@@ -8734,9 +8893,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
<< ".\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return;
}
SmallPtrSet<Value *, 8> Values(E->Scalars.begin(), E->Scalars.end());
@@ -8744,9 +8905,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
return isa<PoisonValue>(V) || Values.contains(V);
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return;
}
}
@@ -8764,9 +8927,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
cast<Instruction>(I)->getOpcode() == S.getOpcode();
})))) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return;
}
@@ -8775,16 +8940,19 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
isa<ScalableVectorType>(
cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return;
}
// Don't handle vectors.
if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
return;
}
@@ -8800,9 +8968,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
if (any_of(VL, [&](Value *V) {
return ScalarsInSplitNodes.contains(V) || isVectorized(V);
})) {
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return true;
}
SmallVector<Value *> Op1, Op2;
@@ -8905,7 +9075,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
SmallVector<Value *> NewVL(VL.size());
copy(Op1, NewVL.begin());
copy(Op2, std::next(NewVL.begin(), Op1.size()));
- auto *TE = newTreeEntry(VL, TreeEntry::SplitVectorize, std::nullopt,
+ auto Invalid = ScheduleBundle::invalid();
+ auto *TE = newTreeEntry(VL, TreeEntry::SplitVectorize, Invalid,
LocalState, UserTreeIdx, {}, ReorderIndices);
LLVM_DEBUG(dbgs() << "SLP: split alternate node.\n"; TE->dump());
auto AddNode = [&](ArrayRef<Value *> Op, unsigned Idx) {
@@ -8915,7 +9086,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
// Build gather node for loads, they will be gathered later.
TE->CombinedEntriesWithIndices.emplace_back(VectorizableTree.size(),
Idx == 0 ? 0 : Op1.size());
- (void)newTreeEntry(Op, TreeEntry::NeedToGather, std::nullopt, S,
+ (void)newTreeEntry(Op, TreeEntry::NeedToGather, Invalid, S,
{TE, Idx});
} else {
TE->CombinedEntriesWithIndices.emplace_back(VectorizableTree.size(),
@@ -9021,9 +9192,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
return;
}
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return;
}
@@ -9033,7 +9206,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
if (EphValues.count(V)) {
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
<< ") is ephemeral.\n");
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
return;
}
}
@@ -9050,9 +9224,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
if (isVectorized(V)) {
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
<< ") is already in tree.\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)){
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return;
}
}
@@ -9062,9 +9238,11 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
for (Value *V : VL) {
if (UserIgnoreList->contains(V)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
- if (TryToFindDuplicates(S))
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ if (TryToFindDuplicates(S)) {
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ }
return;
}
}
@@ -9095,7 +9273,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
// Do not vectorize EH and non-returning blocks, not profitable in most
// cases.
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx);
return;
}
@@ -9113,7 +9292,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
TreeEntry::EntryState State = getScalarsVectorizationState(
S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
if (State == TreeEntry::NeedToGather) {
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
return;
}
@@ -9124,22 +9304,22 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
BlockScheduling &BS = *BSRef;
- std::optional<ScheduleData *> Bundle =
+ std::optional<ScheduleBundle *> BundlePtr =
BS.tryScheduleBundle(UniqueValues, this, S);
#ifdef EXPENSIVE_CHECKS
// Make sure we didn't break any internal invariants
BS.verify();
#endif
- if (!Bundle) {
+ if (!BundlePtr || (*BundlePtr && !*BundlePtr.value())) {
LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
- assert((!BS.getScheduleData(VL0) ||
- !BS.getScheduleData(VL0)->isPartOfBundle()) &&
- "tryScheduleBundle should cancelScheduling on failure");
+ assert((!BS.getScheduleData(VL0) || BS.getScheduleBundles(VL0).empty()) &&
+ "tryScheduleBundle should not create bundle on failure");
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() &&
TrySplitNode(SmallNodeSize, S))
return;
- newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+ auto Invalid = ScheduleBundle::invalid();
+ newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
@@ -9147,6 +9327,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
registerNonVectorizableLoads(VL);
return;
}
+ ScheduleBundle Empty;
+ ScheduleBundle &Bundle = BundlePtr.value() ? *BundlePtr.value() : Empty;
LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
unsigned ShuffleOrOp =
@@ -13351,8 +13533,9 @@ template <typename T> struct ShuffledInsertData {
};
} // namespace
-InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
- InstructionCost Cost = 0;
+InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals,
+ InstructionCost ReductionCost) {
+ InstructionCost Cost = ReductionCost;
LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n");
@@ -13396,6 +13579,12 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
<< "SLP: Current total cost = " << Cost << "\n");
}
+ if (Cost >= -SLPCostThreshold &&
+ none_of(ExternalUses, [](const ExternalUser &EU) {
+ return isa_and_nonnull<InsertElementInst>(EU.User);
+ }))
+ return Cost;
+
SmallPtrSet<Value *, 16> ExtractCostCalculated;
InstructionCost ExtractCost = 0;
SmallVector<ShuffledInsertData<const TreeEntry *>> ShuffledInserts;
@@ -14855,10 +15044,16 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
Value *V = E->isOneOf(E->Scalars.back());
if (doesNotNeedToBeScheduled(V))
V = *find_if_not(E->Scalars, doesNotNeedToBeScheduled);
- auto *Bundle = BlocksSchedules[BB]->getScheduleData(V);
- if (Bundle && Bundle->isPartOfBundle())
- for (; Bundle; Bundle = Bundle->NextInBundle)
- Res = Bundle->Inst;
+ if (ArrayRef<ScheduleBundle *> Bundles =
+ BlocksSchedules[BB]->getScheduleBundles(V);
+ !Bundles.empty()) {
+ const auto *It = find_if(
+ Bundles, [&](ScheduleBundle *B) { return B->getTreeEntry() == E; });
+ assert(It != Bundles.end() && "Failed to find bundle");
+ Res = (*It)->getBundle().back()->getInst();
+ return *Res;
+ }
+ assert(E->getOpcode() == Instruction::PHI && "Expected PHI");
}
// LastInst can still be null at this point if there's either not an entry
@@ -18036,36 +18231,30 @@ void BoUpSLP::optimizeGatherSequence() {
GatherShuffleExtractSeq.clear();
}
-BoUpSLP::ScheduleData *
+BoUpSLP::ScheduleBundle &
BoUpSLP::BlockScheduling::buildBundle(ArrayRef<Value *> VL) {
- ScheduleData *Bundle = nullptr;
- ScheduleData *PrevInBundle = nullptr;
+ auto &BundlePtr =
+ ScheduledBundlesList.emplace_back(std::make_unique<ScheduleBundle>());
for (Value *V : VL) {
if (doesNotNeedToBeScheduled(V))
continue;
ScheduleData *BundleMember = getScheduleData(V);
- assert(BundleMember &&
- "no ScheduleData for bundle member "
- "(maybe not in same basic block)");
- assert(BundleMember->isSchedulingEntity() &&
- "bundle member already part of other bundle");
- if (PrevInBundle) {
- PrevInBundle->NextInBundle = BundleMember;
- } else {
- Bundle = BundleMember;
- }
-
+ assert(BundleMember && "no ScheduleData for bundle member "
+ "(maybe not in same basic block)");
// Group the instructions to a bundle.
- BundleMember->FirstInBundle = Bundle;
- PrevInBundle = BundleMember;
- }
- assert(Bundle && "Failed to find schedule bundle");
- return Bundle;
+ BundlePtr->add(BundleMember);
+ ScheduledBundles.try_emplace(cast<Instruction>(V))
+ .first->getSecond()
+ .push_back(BundlePtr.get());
+ }
+ assert(BundlePtr.get() && *BundlePtr.get() &&
+ "Failed to find schedule bundle");
+ return *BundlePtr.get();
}
// Groups the instructions to a bundle (which is then a single scheduling entity)
// and schedules instructions until the bundle gets ready.
-std::optional<BoUpSLP::ScheduleData *>
+std::optional<BoUpSLP::ScheduleBundle *>
BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
const InstructionsState &S) {
// No need to schedule PHIs, insertelement, extractelement and extractvalue
@@ -18078,23 +18267,23 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
Instruction *OldScheduleEnd = ScheduleEnd;
LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.getMainOp() << "\n");
- auto TryScheduleBundleImpl = [this, OldScheduleEnd, SLP](bool ReSchedule,
- ScheduleData *Bundle) {
+ auto TryScheduleBundleImpl = [=](bool ReSchedule, ScheduleBundle &Bundle) {
// The scheduling region got new instructions at the lower end (or it is a
// new region for the first bundle). This makes it necessary to
// recalculate all dependencies.
// It is seldom that this needs to be done a second time after adding the
// initial bundle to the region.
- if (ScheduleEnd != OldScheduleEnd) {
- for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode())
+ if (OldScheduleEnd && ScheduleEnd != OldScheduleEnd) {
+ for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
if (ScheduleData *SD = getScheduleData(I))
SD->clearDependencies();
+ }
ReSchedule = true;
}
- if (Bundle) {
- LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle
+ if (Bundle && !Bundle.getBundle().empty()) {
+ LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << Bundle
<< " in block " << BB->getName() << "\n");
- calculateDependencies(Bundle, /*InsertInReadyList=*/true, SLP);
+ calculateDependencies(Bundle, /*InsertInReadyList=*/!ReSchedule, SLP);
}
if (ReSchedule) {
@@ -18105,13 +18294,22 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
// Now try to schedule the new bundle or (if no bundle) just calculate
// dependencies. As soon as the bundle is "ready" it means that there are no
// cyclic dependencies and we can schedule it. Note that's important that we
- // don't "schedule" the bundle yet (see cancelScheduling).
- while (((!Bundle && ReSchedule) || (Bundle && !Bundle->isReady())) &&
+ // don't "schedule" the bundle yet.
+ SmallPtrSet<const ScheduleBundle *, 16> Visited;
+ while (((!Bundle && ReSchedule) || (Bundle && !Bundle.isReady())) &&
!ReadyInsts.empty()) {
- ScheduleData *Picked = ReadyInsts.pop_back_val();
- assert(Picked->isSchedulingEntity() && Picked->isReady() &&
+ ScheduleEntity *Picked = ReadyInsts.pop_back_val();
+ const auto *PickedBundle = dyn_cast<ScheduleBundle>(Picked);
+ if (PickedBundle && !Visited.insert(PickedBundle).second) {
+ assert(PickedBundle->isScheduled() && "bundle must be scheduled");
+ continue;
+ }
+ assert((PickedBundle ? PickedBundle->isReady()
+ : cast<ScheduleData>(Picked)->isReady()) &&
"must be ready to schedule");
schedule(Picked, ReadyInsts);
+ if (Picked == &Bundle)
+ break;
}
};
@@ -18127,7 +18325,8 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
// Otherwise the compiler may crash trying to incorrectly calculate
// dependencies and emit instruction in the wrong order at the actual
// scheduling.
- TryScheduleBundleImpl(/*ReSchedule=*/false, nullptr);
+ ScheduleBundle Invalid = ScheduleBundle::invalid();
+ TryScheduleBundleImpl(/*ReSchedule=*/false, Invalid);
return std::nullopt;
}
}
@@ -18143,8 +18342,11 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
// Make sure we don't leave the pieces of the bundle in the ready list when
// whole bundle might not be ready.
ReadyInsts.remove(BundleMember);
+ if (ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(V);
+ !Bundles.empty())
+ for_each(Bundles, [&](ScheduleBundle *B) { ReadyInsts.remove(B); });
- if (!BundleMember->IsScheduled)
+ if (!BundleMember->isScheduled())
continue;
// A bundle member was scheduled as single instruction before and now
// needs to be scheduled as part of the bundle. We just get rid of the
@@ -18154,48 +18356,22 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
ReSchedule = true;
}
- auto *Bundle = buildBundle(VL);
+ ScheduleBundle &Bundle = buildBundle(VL);
TryScheduleBundleImpl(ReSchedule, Bundle);
- if (!Bundle->isReady()) {
- cancelScheduling(VL, S.getMainOp());
+ if (!Bundle.isReady()) {
+ for (ScheduleData *BD : Bundle.getBundle()) {
+ if (BD->isReady())
+ ReadyInsts.insert(BD);
+ }
+ ScheduledBundlesList.pop_back();
+ for (Value *V : VL) {
+ if (doesNotNeedToBeScheduled(V))
+ continue;
+ ScheduledBundles.find(cast<Instruction>(V))->getSecond().pop_back();
+ }
return std::nullopt;
}
- return Bundle;
-}
-
-void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
- Value *OpValue) {
- if (isa<PHINode>(OpValue) || isVectorLikeInstWithConstOps(OpValue) ||
- doesNotNeedToSchedule(VL))
- return;
-
- if (doesNotNeedToBeScheduled(OpValue))
- OpValue = *find_if_not(VL, doesNotNeedToBeScheduled);
- ScheduleData *Bundle = getScheduleData(OpValue);
- LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
- assert(!Bundle->IsScheduled &&
- "Can't cancel bundle which is already scheduled");
- assert(Bundle->isSchedulingEntity() &&
- (Bundle->isPartOfBundle() || needToScheduleSingleInstruction(VL)) &&
- "tried to unbundle something which is not a bundle");
-
- // Remove the bundle from the ready list.
- if (Bundle->isReady())
- ReadyInsts.remove(Bundle);
-
- // Un-bundle: make single instructions out of the bundle.
- ScheduleData *BundleMember = Bundle;
- while (BundleMember) {
- assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
- BundleMember->FirstInBundle = BundleMember;
- ScheduleData *Next = BundleMember->NextInBundle;
- BundleMember->NextInBundle = nullptr;
- BundleMember->TE = nullptr;
- if (BundleMember->unscheduledDepsInBundle() == 0) {
- ReadyInsts.insert(BundleMember);
- }
- BundleMember = Next;
- }
+ return &Bundle;
}
BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
@@ -18302,7 +18478,7 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
Intrinsic::pseudoprobe))) {
// Update the linked list of memory accessing instructions.
if (CurrentLoadStore) {
- CurrentLoadStore->NextLoadStore = SD;
+ CurrentLoadStore->setNextLoadStore(SD);
} else {
FirstLoadStoreInRegion = SD;
}
@@ -18315,182 +18491,199 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
}
if (NextLoadStore) {
if (CurrentLoadStore)
- CurrentLoadStore->NextLoadStore = NextLoadStore;
+ CurrentLoadStore->setNextLoadStore(NextLoadStore);
} else {
LastLoadStoreInRegion = CurrentLoadStore;
}
}
-void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
+void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
bool InsertInReadyList,
BoUpSLP *SLP) {
- assert(SD->isSchedulingEntity());
+ SmallVector<ScheduleData *> WorkList;
+ auto ProcessNode = [&](ScheduleData *BundleMember) {
+ assert(!BundleMember->hasValidDependencies() && "invalid deps expected.");
+ BundleMember->initDependencies();
+ BundleMember->resetUnscheduledDeps();
+ // Handle def-use chain dependencies.
+ for (User *U : BundleMember->getInst()->users()) {
+ if (ScheduleData *UseSD = getScheduleData(U)) {
+ BundleMember->incDependencies();
+ if (!UseSD->isScheduled())
+ BundleMember->incrementUnscheduledDeps(1);
+ WorkList.push_back(UseSD);
+ }
+ }
- SmallVector<ScheduleData *, 10> WorkList;
- WorkList.push_back(SD);
+ auto MakeControlDependent = [&](Instruction *I) {
+ auto *DepDest = getScheduleData(I);
+ assert(DepDest && "must be in schedule window");
+ DepDest->addControlDependency(BundleMember);
+ BundleMember->incDependencies();
+ if (!DepDest->isScheduled())
+ BundleMember->incrementUnscheduledDeps(1);
+ WorkList.push_back(DepDest);
+ };
- while (!WorkList.empty()) {
- ScheduleData *SD = WorkList.pop_back_val();
- for (ScheduleData *BundleMember = SD; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- assert(isInSchedulingRegion(BundleMember));
- if (BundleMember->hasValidDependencies())
- continue;
+ // Any instruction which isn't safe to speculate at the beginning of the
+ // block is control depend on any early exit or non-willreturn call
+ // which proceeds it.
+ if (!isGuaranteedToTransferExecutionToSuccessor(BundleMember->getInst())) {
+ for (Instruction *I = BundleMember->getInst()->getNextNode();
+ I != ScheduleEnd; I = I->getNextNode()) {
+ if (isSafeToSpeculativelyExecute(I, &*BB->begin(), SLP->AC))
+ continue;
- LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember
- << "\n");
- BundleMember->Dependencies = 0;
- BundleMember->resetUnscheduledDeps();
-
- // Handle def-use chain dependencies.
- for (User *U : BundleMember->Inst->users()) {
- if (ScheduleData *UseSD = getScheduleData(cast<Instruction>(U))) {
- BundleMember->Dependencies++;
- ScheduleData *DestBundle = UseSD->FirstInBundle;
- if (!DestBundle->IsScheduled)
- BundleMember->incrementUnscheduledDeps(1);
- if (!DestBundle->hasValidDependencies())
- WorkList.push_back(DestBundle);
- }
- }
+ // Add the dependency
+ MakeControlDependent(I);
- auto MakeControlDependent = [&](Instruction *I) {
- auto *DepDest = getScheduleData(I);
- assert(DepDest && "must be in schedule window");
- DepDest->ControlDependencies.push_back(BundleMember);
- BundleMember->Dependencies++;
- ScheduleData *DestBundle = DepDest->FirstInBundle;
- if (!DestBundle->IsScheduled)
- BundleMember->incrementUnscheduledDeps(1);
- if (!DestBundle->hasValidDependencies())
- WorkList.push_back(DestBundle);
- };
+ if (!isGuaranteedToTransferExecutionToSuccessor(I))
+ // Everything past here must be control dependent on I.
+ break;
+ }
+ }
- // Any instruction which isn't safe to speculate at the beginning of the
- // block is control dependend on any early exit or non-willreturn call
- // which proceeds it.
- if (!isGuaranteedToTransferExecutionToSuccessor(BundleMember->Inst)) {
- for (Instruction *I = BundleMember->Inst->getNextNode();
+ if (RegionHasStackSave) {
+ // If we have an inalloc alloca instruction, it needs to be scheduled
+ // after any preceeding stacksave. We also need to prevent any alloca
+ // from reordering above a preceeding stackrestore.
+ if (match(BundleMember->getInst(), m_Intrinsic<Intrinsic::stacksave>()) ||
+ match(BundleMember->getInst(),
+ m_Intrinsic<Intrinsic::stackrestore>())) {
+ for (Instruction *I = BundleMember->getInst()->getNextNode();
I != ScheduleEnd; I = I->getNextNode()) {
- if (isSafeToSpeculativelyExecute(I, &*BB->begin(), SLP->AC))
+ if (match(I, m_Intrinsic<Intrinsic::stacksave>()) ||
+ match(I, m_Intrinsic<Intrinsic::stackrestore>()))
+ // Any allocas past here must be control dependent on I, and I
+ // must be memory dependend on BundleMember->Inst.
+ break;
+
+ if (!isa<AllocaInst>(I))
continue;
// Add the dependency
MakeControlDependent(I);
-
- if (!isGuaranteedToTransferExecutionToSuccessor(I))
- // Everything past here must be control dependent on I.
- break;
}
}
- if (RegionHasStackSave) {
- // If we have an inalloc alloca instruction, it needs to be scheduled
- // after any preceeding stacksave. We also need to prevent any alloca
- // from reordering above a preceeding stackrestore.
- if (match(BundleMember->Inst, m_Intrinsic<Intrinsic::stacksave>()) ||
- match(BundleMember->Inst, m_Intrinsic<Intrinsic::stackrestore>())) {
- for (Instruction *I = BundleMember->Inst->getNextNode();
- I != ScheduleEnd; I = I->getNextNode()) {
- if (match(I, m_Intrinsic<Intrinsic::stacksave>()) ||
- match(I, m_Intrinsic<Intrinsic::stackrestore>()))
- // Any allocas past here must be control dependent on I, and I
- // must be memory dependend on BundleMember->Inst.
- break;
-
- if (!isa<AllocaInst>(I))
- continue;
+ // In addition to the cases handle just above, we need to prevent
+ // allocas and loads/stores from moving below a stacksave or a
+ // stackrestore. Avoiding moving allocas below stackrestore is currently
+ // thought to be conservatism. Moving loads/stores below a stackrestore
+ // can lead to incorrect code.
+ if (isa<AllocaInst>(BundleMember->getInst()) ||
+ BundleMember->getInst()->mayReadOrWriteMemory()) {
+ for (Instruction *I = BundleMember->getInst()->getNextNode();
+ I != ScheduleEnd; I = I->getNextNode()) {
+ if (!match(I, m_Intrinsic<Intrinsic::stacksave>()) &&
+ !match(I, m_Intrinsic<Intrinsic::stackrestore>()))
+ continue;
- // Add the dependency
- MakeControlDependent(I);
- }
+ // Add the dependency
+ MakeControlDependent(I);
+ break;
}
+ }
+ }
- // In addition to the cases handle just above, we need to prevent
- // allocas and loads/stores from moving below a stacksave or a
- // stackrestore. Avoiding moving allocas below stackrestore is currently
- // thought to be conservatism. Moving loads/stores below a stackrestore
- // can lead to incorrect code.
- if (isa<AllocaInst>(BundleMember->Inst) ||
- BundleMember->Inst->mayReadOrWriteMemory()) {
- for (Instruction *I = BundleMember->Inst->getNextNode();
- I != ScheduleEnd; I = I->getNextNode()) {
- if (!match(I, m_Intrinsic<Intrinsic::stacksave>()) &&
- !match(I, m_Intrinsic<Intrinsic::stackrestore>()))
- continue;
-
- // Add the dependency
- MakeControlDependent(I);
- break;
- }
- }
+ // Handle the memory dependencies (if any).
+ ScheduleData *NextLoadStore = BundleMember->getNextLoadStore();
+ if (!NextLoadStore)
+ return;
+ Instruction *SrcInst = BundleMember->getInst();
+ assert(SrcInst->mayReadOrWriteMemory() &&
+ "NextLoadStore list for non memory effecting bundle?");
+ MemoryLocation SrcLoc = getLocation(SrcInst);
+ bool SrcMayWrite = SrcInst->mayWriteToMemory();
+ unsigned NumAliased = 0;
+ unsigned DistToSrc = 1;
+ bool IsNonSimpleSrc = !SrcLoc.Ptr || !isSimple(SrcInst);
+
+ for (ScheduleData *DepDest = NextLoadStore; DepDest;
+ DepDest = DepDest->getNextLoadStore()) {
+ assert(isInSchedulingRegion(DepDest) && "Expected to be in region");
+
+ // We have two limits to reduce the complexity:
+ // 1) AliasedCheckLimit: It's a small limit to reduce calls to
+ // SLP->isAliased (which is the expensive part in this loop).
+ // 2) MaxMemDepDistance: It's for very large blocks and it aborts
+ // the whole loop (even if the loop is fast, it's quadratic).
+ // It's important for the loop break condition (see below) to
+ // check this limit even between two read-only instructions.
+ if (DistToSrc >= MaxMemDepDistance ||
+ ((SrcMayWrite || DepDest->getInst()->mayWriteToMemory()) &&
+ (IsNonSimpleSrc || NumAliased >= AliasedCheckLimit ||
+ SLP->isAliased(SrcLoc, SrcInst, DepDest->getInst())))) {
+
+ // We increment the counter only if the locations are aliased
+ // (instead of counting all alias checks). This gives a better
+ // balance between reduced runtime and accurate dependencies.
+ NumAliased++;
+
+ DepDest->addMemoryDependency(BundleMember);
+ BundleMember->incDependencies();
+ if (!DepDest->isScheduled())
+ BundleMember->incrementUnscheduledDeps(1);
+ WorkList.push_back(DepDest);
}
- // Handle the memory dependencies (if any).
- ScheduleData *DepDest = BundleMember->NextLoadStore;
- if (!DepDest)
- continue;
- Instruction *SrcInst = BundleMember->Inst;
- assert(SrcInst->mayReadOrWriteMemory() &&
- "NextLoadStore list for non memory effecting bundle?");
- MemoryLocation SrcLoc = getLocation(SrcInst);
- bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
- unsigned NumAliased = 0;
- unsigned DistToSrc = 1;
-
- for (; DepDest; DepDest = DepDest->NextLoadStore) {
- assert(isInSchedulingRegion(DepDest));
-
- // We have two limits to reduce the complexity:
- // 1) AliasedCheckLimit: It's a small limit to reduce calls to
- // SLP->isAliased (which is the expensive part in this loop).
- // 2) MaxMemDepDistance: It's for very large blocks and it aborts
- // the whole loop (even if the loop is fast, it's quadratic).
- // It's important for the loop break condition (see below) to
- // check this limit even between two read-only instructions.
- if (DistToSrc >= MaxMemDepDistance ||
- ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
- (NumAliased >= AliasedCheckLimit ||
- SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
-
- // We increment the counter only if the locations are aliased
- // (instead of counting all alias checks). This gives a better
- // balance between reduced runtime and accurate dependencies.
- NumAliased++;
-
- DepDest->MemoryDependencies.push_back(BundleMember);
- BundleMember->Dependencies++;
- ScheduleData *DestBundle = DepDest->FirstInBundle;
- if (!DestBundle->IsScheduled) {
- BundleMember->incrementUnscheduledDeps(1);
- }
- if (!DestBundle->hasValidDependencies()) {
- WorkList.push_back(DestBundle);
- }
- }
+ // Example, explaining the loop break condition: Let's assume our
+ // starting instruction is i0 and MaxMemDepDistance = 3.
+ //
+ // +--------v--v--v
+ // i0,i1,i2,i3,i4,i5,i6,i7,i8
+ // +--------^--^--^
+ //
+ // MaxMemDepDistance let us stop alias-checking at i3 and we add
+ // dependencies from i0 to i3,i4,.. (even if they are not aliased).
+ // Previously we already added dependencies from i3 to i6,i7,i8
+ // (because of MaxMemDepDistance). As we added a dependency from
+ // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
+ // and we can abort this loop at i6.
+ if (DistToSrc >= 2 * MaxMemDepDistance)
+ break;
+ DistToSrc++;
+ }
+ };
- // Example, explaining the loop break condition: Let's assume our
- // starting instruction is i0 and MaxMemDepDistance = 3.
- //
- // +--------v--v--v
- // i0,i1,i2,i3,i4,i5,i6,i7,i8
- // +--------^--^--^
- //
- // MaxMemDepDistance let us stop alias-checking at i3 and we add
- // dependencies from i0 to i3,i4,.. (even if they are not aliased).
- // Previously we already added dependencies from i3 to i6,i7,i8
- // (because of MaxMemDepDistance). As we added a dependency from
- // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
- // and we can abort this loop at i6.
- if (DistToSrc >= 2 * MaxMemDepDistance)
- break;
- DistToSrc++;
+ WorkList.push_back(Bundle.getBundle().front());
+ SmallPtrSet<ScheduleBundle *, 16> Visited;
+ while (!WorkList.empty()) {
+ ScheduleData *SD = WorkList.pop_back_val();
+ ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(SD->getInst());
+ if (!Bundles.empty()) {
+ for (ScheduleBundle *Bundle : Bundles) {
+ if (!Visited.insert(Bundle).second || Bundle->hasValidDependencies())
+ continue;
+ assert(isInSchedulingRegion(*Bundle) &&
+ "ScheduleData not in scheduling region");
+ for (ScheduleData *BundleMember : Bundle->getBundle()) {
+ if (BundleMember->hasValidDependencies())
+ continue;
+ LLVM_DEBUG(dbgs()
+ << "SLP: update deps of " << *BundleMember << "\n");
+ ProcessNode(BundleMember);
+ }
}
+ } else if (!SD->hasValidDependencies()) {
+ LLVM_DEBUG(dbgs() << "SLP: update deps of " << *SD << "\n");
+ ProcessNode(SD);
}
if (InsertInReadyList && SD->isReady()) {
+ if (!Bundles.empty()) {
+ for (ScheduleBundle *Bundle : Bundles) {
+ assert(isInSchedulingRegion(*Bundle) &&
+ "ScheduleData not in scheduling region");
+ if (Bundle->isReady()) {
+ ReadyInsts.insert(Bundle);
+ LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *Bundle
+ << "\n");
+ }
+ }
+ continue;
+ }
ReadyInsts.insert(SD);
- LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst
- << "\n");
+ LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD << "\n");
}
}
}
@@ -18502,7 +18695,7 @@ void BoUpSLP::BlockScheduling::resetSchedule() {
if (ScheduleData *SD = getScheduleData(I)) {
assert(isInSchedulingRegion(SD) &&
"ScheduleData not in scheduling region");
- SD->IsScheduled = false;
+ SD->setScheduled(/*Scheduled=*/false);
SD->resetUnscheduledDeps();
}
}
@@ -18528,28 +18721,34 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
// WARNING: If changing this order causes a correctness issue, that means
// there is some missing dependence edge in the schedule data graph.
struct ScheduleDataCompare {
- bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
- return SD2->SchedulingPriority < SD1->SchedulingPriority;
+ bool operator()(const ScheduleEntity *SD1,
+ const ScheduleEntity *SD2) const {
+ return SD2->getSchedulingPriority() < SD1->getSchedulingPriority();
}
};
- std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
+ std::set<ScheduleEntity *, ScheduleDataCompare> ReadyInsts;
// Ensure that all dependency data is updated (for nodes in the sub-graph)
// and fill the ready-list with initial instructions.
int Idx = 0;
for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
I = I->getNextNode()) {
+ ArrayRef<ScheduleBundle *> Bundles = BS->getScheduleBundles(I);
+ if (!Bundles.empty()) {
+ for (ScheduleBundle *Bundle : Bundles) {
+ Bundle->setSchedulingPriority(Idx++);
+ if (!Bundle->hasValidDependencies())
+ BS->calculateDependencies(*Bundle, /*InsertInReadyList=*/false, this);
+ }
+ continue;
+ }
if (ScheduleData *SD = BS->getScheduleData(I)) {
- [[maybe_unused]] ArrayRef<TreeEntry *> SDTEs = getTreeEntries(SD->Inst);
- assert((isVectorLikeInstWithConstOps(SD->Inst) ||
- SD->isPartOfBundle() ==
- (!SDTEs.empty() &&
- !doesNotNeedToSchedule(SDTEs.front()->Scalars))) &&
+ [[maybe_unused]] ArrayRef<TreeEntry *> SDTEs = getTreeEntries(I);
+ assert((isVectorLikeInstWithConstOps(SD->getInst()) || SDTEs.empty() ||
+ doesNotNeedToSchedule(SDTEs.front()->Scalars)) &&
"scheduler and vectorizer bundle mismatch");
- SD->FirstInBundle->SchedulingPriority = Idx++;
-
- if (SD->isSchedulingEntity() && SD->isPartOfBundle())
- BS->calculateDependencies(SD, false, this);
+ SD->setSchedulingPriority(Idx++);
+ continue;
}
}
BS->initialFillReadyList(ReadyInsts);
@@ -18558,19 +18757,25 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
// Do the "real" scheduling.
while (!ReadyInsts.empty()) {
- ScheduleData *Picked = *ReadyInsts.begin();
+ auto *Picked = *ReadyInsts.begin();
ReadyInsts.erase(ReadyInsts.begin());
// Move the scheduled instruction(s) to their dedicated places, if not
// there yet.
- for (ScheduleData *BundleMember = Picked; BundleMember;
- BundleMember = BundleMember->NextInBundle) {
- Instruction *PickedInst = BundleMember->Inst;
+ if (auto *Bundle = dyn_cast<ScheduleBundle>(Picked)) {
+ for (const ScheduleData *BundleMember : Bundle->getBundle()) {
+ Instruction *PickedInst = BundleMember->getInst();
+ if (PickedInst->getNextNonDebugInstruction() != LastScheduledInst)
+ PickedInst->moveAfter(LastScheduledInst->getPrevNode());
+ LastScheduledInst = PickedInst;
+ }
+ } else {
+ auto *SD = cast<ScheduleData>(Picked);
+ Instruction *PickedInst = SD->getInst();
if (PickedInst->getNextNonDebugInstruction() != LastScheduledInst)
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
}
-
BS->schedule(Picked, ReadyInsts);
}
@@ -18581,10 +18786,13 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
// Check that all schedulable entities got scheduled
- for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; I = I->getNextNode()) {
- ScheduleData *SD = BS->getScheduleData(I);
- if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies())
- assert(SD->IsScheduled && "must be scheduled at this point");
+ for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
+ I = I->getNextNode()) {
+ ArrayRef<ScheduleBundle *> Bundles = BS->getScheduleBundles(I);
+ if (!Bundles.empty()) {
+ for (ScheduleBundle *Bundle : Bundles)
+ assert(Bundle->isScheduled() && "must be scheduled at this point");
+ }
}
#endif
@@ -21201,10 +21409,9 @@ class HorizontalReduction {
V.computeMinimumValueSizes();
// Estimate cost.
- InstructionCost TreeCost = V.getTreeCost(VL);
InstructionCost ReductionCost =
getReductionCost(TTI, VL, IsCmpSelMinMax, RdxFMF, V);
- InstructionCost Cost = TreeCost + ReductionCost;
+ InstructionCost Cost = V.getTreeCost(VL, ReductionCost);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
<< " for reduction\n");
if (!Cost.isValid())
More information about the llvm-commits
mailing list