[llvm] 50770d8 - [MCA] Refactor the InOrderIssueStage stage. NFCI
Andrea Di Biagio via llvm-commits
llvm-commits at lists.llvm.org
Thu May 27 14:36:04 PDT 2021
Author: Andrea Di Biagio
Date: 2021-05-27T22:28:04+01:00
New Revision: 50770d8de56068312bd0e1baa63e86912ce2b631
URL: https://github.com/llvm/llvm-project/commit/50770d8de56068312bd0e1baa63e86912ce2b631
DIFF: https://github.com/llvm/llvm-project/commit/50770d8de56068312bd0e1baa63e86912ce2b631.diff
LOG: [MCA] Refactor the InOrderIssueStage stage. NFCI
Moved the logic that checks for RAW hazards from the InOrderIssueStage to the
RegisterFile.
Changed how the InOrderIssueStage keeps track of backend stalls. Stall events
are now generated from method notifyStallEvent().
No functional change intended.
Added:
Modified:
llvm/include/llvm/MCA/HWEventListener.h
llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
llvm/include/llvm/MCA/Stages/ExecuteStage.h
llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
llvm/include/llvm/MCA/Stages/InstructionTables.h
llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
llvm/lib/MCA/Stages/ExecuteStage.cpp
llvm/lib/MCA/Stages/InOrderIssueStage.cpp
llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/MCA/HWEventListener.h b/llvm/include/llvm/MCA/HWEventListener.h
index e11d06de2b2ed..aeb5ab14dcdee 100644
--- a/llvm/include/llvm/MCA/HWEventListener.h
+++ b/llvm/include/llvm/MCA/HWEventListener.h
@@ -59,14 +59,15 @@ class HWInstructionEvent {
const InstRef &IR;
};
+using ResourceRef = std::pair<uint64_t, uint64_t>;
+using ResourceUse = std::pair<ResourceRef, ResourceCycles>;
+
class HWInstructionIssuedEvent : public HWInstructionEvent {
public:
- using ResourceRef = std::pair<uint64_t, uint64_t>;
- HWInstructionIssuedEvent(const InstRef &IR,
- ArrayRef<std::pair<ResourceRef, ResourceCycles>> UR)
+ HWInstructionIssuedEvent(const InstRef &IR, ArrayRef<ResourceUse> UR)
: HWInstructionEvent(HWInstructionEvent::Issued, IR), UsedResources(UR) {}
- ArrayRef<std::pair<ResourceRef, ResourceCycles>> UsedResources;
+ ArrayRef<ResourceUse> UsedResources;
};
class HWInstructionDispatchedEvent : public HWInstructionEvent {
@@ -165,7 +166,6 @@ class HWEventListener {
virtual void onEvent(const HWStallEvent &Event) {}
virtual void onEvent(const HWPressureEvent &Event) {}
- using ResourceRef = std::pair<uint64_t, uint64_t>;
virtual void onResourceAvailable(const ResourceRef &RRef) {}
// Events generated by the Scheduler when buffered resources are
diff --git a/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h b/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
index 947d656aa2979..1b811978dd76e 100644
--- a/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
+++ b/llvm/include/llvm/MCA/HardwareUnits/RegisterFile.h
@@ -236,6 +236,17 @@ class RegisterFile : public HardwareUnit {
void collectWrites(const MCSubtargetInfo &STI, const ReadState &RS,
SmallVectorImpl<WriteRef> &Writes,
SmallVectorImpl<WriteRef> &CommittedWrites) const;
+ struct RAWHazard {
+ MCPhysReg RegisterID;
+ int CyclesLeft;
+
+ RAWHazard() : RegisterID(), CyclesLeft() {}
+ bool isValid() const { return RegisterID; }
+ bool hasUnknownCycles() const { return CyclesLeft < 0; }
+ };
+
+ RAWHazard checkRAWHazards(const MCSubtargetInfo &STI,
+ const ReadState &RS) const;
// This method updates the register mappings inserting a new register
// definition. This method is also responsible for updating the number of
diff --git a/llvm/include/llvm/MCA/Stages/ExecuteStage.h b/llvm/include/llvm/MCA/Stages/ExecuteStage.h
index 04dd657e9ac81..4c09ca8255ff6 100644
--- a/llvm/include/llvm/MCA/Stages/ExecuteStage.h
+++ b/llvm/include/llvm/MCA/Stages/ExecuteStage.h
@@ -72,9 +72,8 @@ class ExecuteStage final : public Stage {
Error cycleEnd() override;
Error execute(InstRef &IR) override;
- void notifyInstructionIssued(
- const InstRef &IR,
- MutableArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const;
+ void notifyInstructionIssued(const InstRef &IR,
+ MutableArrayRef<ResourceUse> Used) const;
void notifyInstructionExecuted(const InstRef &IR) const;
void notifyInstructionPending(const InstRef &IR) const;
void notifyInstructionReady(const InstRef &IR) const;
diff --git a/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h b/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
index 1a944243db601..01528ab148c37 100644
--- a/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
+++ b/llvm/include/llvm/MCA/Stages/InOrderIssueStage.h
@@ -18,8 +18,6 @@
#include "llvm/MCA/SourceMgr.h"
#include "llvm/MCA/Stages/Stage.h"
-#include <queue>
-
namespace llvm {
struct MCSchedModel;
class MCSubtargetInfo;
@@ -28,6 +26,45 @@ namespace mca {
class RegisterFile;
class ResourceManager;
+struct StallInfo {
+ enum class StallKind { DEFAULT, REGISTER_DEPS, DISPATCH, DELAY };
+
+ InstRef IR;
+ unsigned CyclesLeft;
+ StallKind Kind;
+
+ StallInfo() : IR(), CyclesLeft(), Kind(StallKind::DEFAULT) {}
+
+ bool isValid() const { return (bool)IR; }
+
+ StallKind getStallKind() const { return Kind; }
+ unsigned getCyclesLeft() const { return CyclesLeft; }
+ const InstRef &getInstruction() const { return IR; }
+ InstRef &getInstruction() { return IR; }
+
+ void clear() {
+ IR.invalidate();
+ CyclesLeft = 0;
+ Kind = StallKind::DEFAULT;
+ }
+
+ void update(const InstRef &Inst, unsigned Cycles, StallKind SK) {
+ IR = Inst;
+ CyclesLeft = Cycles;
+ Kind = SK;
+ }
+
+ void cycleEnd() {
+ if (!isValid())
+ return;
+
+ if (!CyclesLeft)
+ return;
+
+ --CyclesLeft;
+ }
+};
+
class InOrderIssueStage final : public Stage {
const MCSchedModel &SM;
const MCSubtargetInfo &STI;
@@ -40,10 +77,7 @@ class InOrderIssueStage final : public Stage {
/// Number of instructions issued in the current cycle.
unsigned NumIssued;
- /// If an instruction cannot execute due to an unmet register or resource
- /// dependency, the it is stalled for StallCyclesLeft.
- InstRef StalledInst;
- unsigned StallCyclesLeft;
+ StallInfo SI;
/// Instruction that is issued in more than 1 cycle.
InstRef CarriedOver;
@@ -61,13 +95,13 @@ class InOrderIssueStage final : public Stage {
InOrderIssueStage(const InOrderIssueStage &Other) = delete;
InOrderIssueStage &operator=(const InOrderIssueStage &Other) = delete;
- /// If IR has an unmet register or resource dependency, canExecute returns
- /// false. StallCycles is set to the number of cycles left before the
- /// instruction can be issued.
- bool canExecute(const InstRef &IR, unsigned *StallCycles) const;
+ /// Returns true if IR can execute during this cycle.
+ /// In case of stall, it updates SI with information about the stalled
+ /// instruction and the stall reason.
+ bool canExecute(const InstRef &IR);
- /// Issue the instruction, or update StallCycles if IR is stalled.
- Error tryIssue(InstRef &IR, unsigned *StallCycles);
+ /// Issue the instruction, or update the StallInfo.
+ Error tryIssue(InstRef &IR);
/// Update status of instructions from IssuedInst.
void updateIssuedInst();
@@ -75,6 +109,18 @@ class InOrderIssueStage final : public Stage {
/// Continue to issue the CarriedOver instruction.
void updateCarriedOver();
+ /// Notifies a stall event to the Stage listener. Stall information is
+ /// obtained from the internal StallInfo field.
+ void notifyStallEvent();
+
+ void notifyInstructionIssued(const InstRef &IR,
+ ArrayRef<ResourceUse> UsedRes);
+ void notifyInstructionDispatched(const InstRef &IR, unsigned Ops,
+ ArrayRef<unsigned> UsedRegs);
+ void notifyInstructionExecuted(const InstRef &IR);
+ void notifyInstructionRetired(const InstRef &IR,
+ ArrayRef<unsigned> FreedRegs);
+
/// Retire instruction once it is executed.
void retireInstruction(InstRef &IR);
@@ -82,8 +128,7 @@ class InOrderIssueStage final : public Stage {
InOrderIssueStage(RegisterFile &PRF, const MCSchedModel &SM,
const MCSubtargetInfo &STI)
: SM(SM), STI(STI), PRF(PRF), RM(std::make_unique<ResourceManager>(SM)),
- NumIssued(0), StallCyclesLeft(0), CarryOver(0), Bandwidth(0),
- LastWriteBackCycle(0) {}
+ NumIssued(), SI(), CarryOver(), Bandwidth(), LastWriteBackCycle() {}
bool isAvailable(const InstRef &) const override;
bool hasWorkToComplete() const override;
diff --git a/llvm/include/llvm/MCA/Stages/InstructionTables.h b/llvm/include/llvm/MCA/Stages/InstructionTables.h
index 30fdf29317899..35b21b0ba94d2 100644
--- a/llvm/include/llvm/MCA/Stages/InstructionTables.h
+++ b/llvm/include/llvm/MCA/Stages/InstructionTables.h
@@ -27,7 +27,7 @@ namespace mca {
class InstructionTables final : public Stage {
const MCSchedModel &SM;
- SmallVector<std::pair<ResourceRef, ResourceCycles>, 4> UsedResources;
+ SmallVector<ResourceUse, 4> UsedResources;
SmallVector<uint64_t, 8> Masks;
public:
diff --git a/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp b/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
index c983b63e102cb..5e38569f8bfb5 100644
--- a/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
+++ b/llvm/lib/MCA/HardwareUnits/RegisterFile.cpp
@@ -30,6 +30,8 @@ WriteRef::WriteRef(unsigned SourceIndex, WriteState *WS)
void WriteRef::commit() {
assert(Write && Write->isExecuted() && "Cannot commit before write back!");
+ RegisterID = Write->getRegisterID();
+ WriteResID = Write->getWriteResourceID();
Write = nullptr;
}
@@ -225,8 +227,8 @@ void RegisterFile::addRegisterWrite(WriteRef Write,
assert(RegID && "Adding an invalid register definition?");
LLVM_DEBUG({
- dbgs() << "RegisterFile: addRegisterWrite [ " << Write.getSourceIndex()
- << ", " << MRI.getName(RegID) << "]\n";
+ dbgs() << "[PRF] addRegisterWrite [ " << Write.getSourceIndex() << ", "
+ << MRI.getName(RegID) << "]\n";
});
// If RenameAs is equal to RegID, then RegID is subject to register renaming
@@ -480,7 +482,7 @@ void RegisterFile::collectWrites(
const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
MCPhysReg RegID = RS.getRegisterID();
assert(RegID && RegID < RegisterMappings.size());
- LLVM_DEBUG(dbgs() << "RegisterFile: collecting writes for register "
+ LLVM_DEBUG(dbgs() << "[PRF] collecting writes for register "
<< MRI.getName(RegID) << '\n');
// Check if this is an alias.
@@ -536,6 +538,57 @@ void RegisterFile::collectWrites(
});
}
+RegisterFile::RAWHazard
+RegisterFile::checkRAWHazards(const MCSubtargetInfo &STI,
+ const ReadState &RS) const {
+ RAWHazard Hazard;
+ SmallVector<WriteRef, 4> Writes;
+ SmallVector<WriteRef, 4> CommittedWrites;
+
+ const MCSchedModel &SM = STI.getSchedModel();
+ const ReadDescriptor &RD = RS.getDescriptor();
+ const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
+
+ collectWrites(STI, RS, Writes, CommittedWrites);
+ for (const WriteRef &WR : Writes) {
+ const WriteState *WS = WR.getWriteState();
+ unsigned WriteResID = WS->getWriteResourceID();
+ int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
+
+ if (WS->getCyclesLeft() == UNKNOWN_CYCLES) {
+ if (Hazard.isValid())
+ continue;
+
+ Hazard.RegisterID = WR.getRegisterID();
+ Hazard.CyclesLeft = UNKNOWN_CYCLES;
+ continue;
+ }
+
+ int CyclesLeft = WS->getCyclesLeft() - ReadAdvance;
+ if (CyclesLeft > 0) {
+ if (Hazard.CyclesLeft < CyclesLeft) {
+ Hazard.RegisterID = WR.getRegisterID();
+ Hazard.CyclesLeft = CyclesLeft;
+ }
+ }
+ }
+ Writes.clear();
+
+ for (const WriteRef &WR : CommittedWrites) {
+ unsigned WriteResID = WR.getWriteResourceID();
+ int NegReadAdvance = -STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
+ int Elapsed = static_cast<int>(getElapsedCyclesFromWriteBack(WR));
+ int CyclesLeft = NegReadAdvance - Elapsed;
+ assert(CyclesLeft > 0 && "Write should not be in the CommottedWrites set!");
+ if (Hazard.CyclesLeft < CyclesLeft) {
+ Hazard.RegisterID = WR.getRegisterID();
+ Hazard.CyclesLeft = CyclesLeft;
+ }
+ }
+
+ return Hazard;
+}
+
void RegisterFile::addRegisterRead(ReadState &RS,
const MCSubtargetInfo &STI) const {
MCPhysReg RegID = RS.getRegisterID();
@@ -608,7 +661,8 @@ unsigned RegisterFile::isAvailable(ArrayRef<MCPhysReg> Regs) const {
// microarchitectural registers in register file #0 was changed by the
// users via flag -reg-file-size. Alternatively, the scheduling model
// specified a too small number of registers for this register file.
- LLVM_DEBUG(dbgs() << "Not enough registers in the register file.\n");
+ LLVM_DEBUG(
+ dbgs() << "[PRF] Not enough registers in the register file.\n");
// FIXME: Normalize the instruction register count to match the
// NumPhysRegs value. This is a highly unusual case, and is not expected
diff --git a/llvm/lib/MCA/Stages/ExecuteStage.cpp b/llvm/lib/MCA/Stages/ExecuteStage.cpp
index 2284ed7f28162..6e021d3d9232a 100644
--- a/llvm/lib/MCA/Stages/ExecuteStage.cpp
+++ b/llvm/lib/MCA/Stages/ExecuteStage.cpp
@@ -51,7 +51,7 @@ bool ExecuteStage::isAvailable(const InstRef &IR) const {
}
Error ExecuteStage::issueInstruction(InstRef &IR) {
- SmallVector<std::pair<ResourceRef, ResourceCycles>, 4> Used;
+ SmallVector<ResourceUse, 4> Used;
SmallVector<InstRef, 4> Pending;
SmallVector<InstRef, 4> Ready;
@@ -203,7 +203,7 @@ Error ExecuteStage::execute(InstRef &IR) {
unsigned NumMicroOps = Inst.getNumMicroOps();
NumDispatchedOpcodes += NumMicroOps;
notifyReservedOrReleasedBuffers(IR, /* Reserved */ true);
-
+
if (!IsReadyInstruction) {
if (Inst.isPending())
notifyInstructionPending(IR);
@@ -250,20 +250,19 @@ void ExecuteStage::notifyResourceAvailable(const ResourceRef &RR) const {
}
void ExecuteStage::notifyInstructionIssued(
- const InstRef &IR,
- MutableArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const {
+ const InstRef &IR, MutableArrayRef<ResourceUse> Used) const {
LLVM_DEBUG({
dbgs() << "[E] Instruction Issued: #" << IR << '\n';
- for (const std::pair<ResourceRef, ResourceCycles> &Resource : Used) {
- assert(Resource.second.getDenominator() == 1 && "Invalid cycles!");
- dbgs() << "[E] Resource Used: [" << Resource.first.first << '.'
- << Resource.first.second << "], ";
- dbgs() << "cycles: " << Resource.second.getNumerator() << '\n';
+ for (const ResourceUse &Use : Used) {
+ assert(Use.second.getDenominator() == 1 && "Invalid cycles!");
+ dbgs() << "[E] Resource Used: [" << Use.first.first << '.'
+ << Use.first.second << "], ";
+ dbgs() << "cycles: " << Use.second.getNumerator() << '\n';
}
});
// Replace resource masks with valid resource processor IDs.
- for (std::pair<ResourceRef, ResourceCycles> &Use : Used)
+ for (ResourceUse &Use : Used)
Use.first.first = HWS.getResourceID(Use.first.first);
notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, Used));
diff --git a/llvm/lib/MCA/Stages/InOrderIssueStage.cpp b/llvm/lib/MCA/Stages/InOrderIssueStage.cpp
index 273685234c85b..bf522a34d66a8 100644
--- a/llvm/lib/MCA/Stages/InOrderIssueStage.cpp
+++ b/llvm/lib/MCA/Stages/InOrderIssueStage.cpp
@@ -12,28 +12,21 @@
//===----------------------------------------------------------------------===//
#include "llvm/MCA/Stages/InOrderIssueStage.h"
-
-#include "llvm/MC/MCSchedule.h"
-#include "llvm/MCA/HWEventListener.h"
#include "llvm/MCA/HardwareUnits/RegisterFile.h"
#include "llvm/MCA/HardwareUnits/ResourceManager.h"
#include "llvm/MCA/HardwareUnits/RetireControlUnit.h"
#include "llvm/MCA/Instruction.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/Error.h"
-
-#include <algorithm>
#define DEBUG_TYPE "llvm-mca"
namespace llvm {
namespace mca {
bool InOrderIssueStage::hasWorkToComplete() const {
- return !IssuedInst.empty() || StalledInst || CarriedOver;
+ return !IssuedInst.empty() || SI.isValid() || CarriedOver;
}
bool InOrderIssueStage::isAvailable(const InstRef &IR) const {
- if (StalledInst || CarriedOver)
+ if (SI.isValid() || CarriedOver)
return false;
const Instruction &Inst = *IR.getInstruction();
@@ -77,87 +70,44 @@ static unsigned findFirstWriteBackCycle(const InstRef &IR) {
/// Return a number of cycles left until register requirements of the
/// instructions are met.
static unsigned checkRegisterHazard(const RegisterFile &PRF,
- const MCSchedModel &SM,
const MCSubtargetInfo &STI,
const InstRef &IR) {
- unsigned StallCycles = 0;
- SmallVector<WriteRef, 4> Writes;
- SmallVector<WriteRef, 4> CommittedWrites;
-
for (const ReadState &RS : IR.getInstruction()->getUses()) {
- const ReadDescriptor &RD = RS.getDescriptor();
- const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
-
- PRF.collectWrites(STI, RS, Writes, CommittedWrites);
- for (const WriteRef &WR : Writes) {
- const WriteState *WS = WR.getWriteState();
- unsigned WriteResID = WS->getWriteResourceID();
- int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
- LLVM_DEBUG(dbgs() << "[E] ReadAdvance for #" << IR << ": " << ReadAdvance
- << '\n');
-
- if (WS->getCyclesLeft() == UNKNOWN_CYCLES) {
- // Try again in the next cycle until the value is known
- StallCycles = std::max(StallCycles, 1U);
- continue;
- }
-
- int CyclesLeft = WS->getCyclesLeft() - ReadAdvance;
- if (CyclesLeft > 0) {
- LLVM_DEBUG(dbgs() << "[E] Register hazard: " << WS->getRegisterID()
- << '\n');
- StallCycles = std::max(StallCycles, (unsigned)CyclesLeft);
- }
- }
- Writes.clear();
-
- for (const WriteRef &WR : CommittedWrites) {
- unsigned WriteResID = WR.getWriteResourceID();
- assert(!WR.getWriteState() && "Should be already committed!");
- assert(WR.hasKnownWriteBackCycle() && "Invalid write!");
- assert(STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID) < 0);
- unsigned ReadAdvance = static_cast<unsigned>(
- -STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID));
- unsigned Elapsed = PRF.getElapsedCyclesFromWriteBack(WR);
- assert(Elapsed < ReadAdvance && "Should not have been added to the set!");
- unsigned CyclesLeft = (ReadAdvance - Elapsed);
- StallCycles = std::max(StallCycles, CyclesLeft);
- }
+ RegisterFile::RAWHazard Hazard = PRF.checkRAWHazards(STI, RS);
+ if (Hazard.isValid())
+ return Hazard.hasUnknownCycles() ? 1U : Hazard.CyclesLeft;
}
- return StallCycles;
+ return 0;
}
-bool InOrderIssueStage::canExecute(const InstRef &IR,
- unsigned *StallCycles) const {
- *StallCycles = 0;
-
- if (unsigned RegStall = checkRegisterHazard(PRF, SM, STI, IR)) {
- *StallCycles = RegStall;
- // FIXME: add a parameter to HWStallEvent to indicate a number of cycles.
- for (unsigned I = 0; I < RegStall; ++I) {
- notifyEvent<HWStallEvent>(
- HWStallEvent(HWStallEvent::RegisterFileStall, IR));
- notifyEvent<HWPressureEvent>(
- HWPressureEvent(HWPressureEvent::REGISTER_DEPS, IR));
- }
- } else if (hasResourceHazard(*RM, IR)) {
- *StallCycles = 1;
- notifyEvent<HWStallEvent>(
- HWStallEvent(HWStallEvent::DispatchGroupStall, IR));
- notifyEvent<HWPressureEvent>(
- HWPressureEvent(HWPressureEvent::RESOURCES, IR));
- } else if (LastWriteBackCycle) {
+bool InOrderIssueStage::canExecute(const InstRef &IR) {
+ assert(!SI.getCyclesLeft() && "Should not have reached this code!");
+ assert(!SI.isValid() && "Should not have reached this code!");
+
+ if (unsigned Cycles = checkRegisterHazard(PRF, STI, IR)) {
+ SI.update(IR, Cycles, StallInfo::StallKind::REGISTER_DEPS);
+ return false;
+ }
+
+ if (hasResourceHazard(*RM, IR)) {
+ SI.update(IR, /* delay */ 1, StallInfo::StallKind::DISPATCH);
+ return false;
+ }
+
+ if (LastWriteBackCycle) {
if (!IR.getInstruction()->getDesc().RetireOOO) {
unsigned NextWriteBackCycle = findFirstWriteBackCycle(IR);
- // Delay the instruction to ensure that writes occur in program order
+ // Delay the instruction to ensure that writes happen in program order.
if (NextWriteBackCycle < LastWriteBackCycle) {
- *StallCycles = LastWriteBackCycle - NextWriteBackCycle;
+ SI.update(IR, LastWriteBackCycle - NextWriteBackCycle,
+ StallInfo::StallKind::DELAY);
+ return false;
}
}
}
- return *StallCycles == 0;
+ return true;
}
static void addRegisterReadWrite(RegisterFile &PRF, Instruction &IS,
@@ -173,47 +123,53 @@ static void addRegisterReadWrite(RegisterFile &PRF, Instruction &IS,
PRF.addRegisterWrite(WriteRef(SourceIndex, &WS), UsedRegs);
}
-static void notifyInstructionIssue(
- const InstRef &IR,
- const SmallVectorImpl<std::pair<ResourceRef, ResourceCycles>> &UsedRes,
- const Stage &S) {
-
- S.notifyEvent<HWInstructionEvent>(
+void InOrderIssueStage::notifyInstructionIssued(const InstRef &IR,
+ ArrayRef<ResourceUse> UsedRes) {
+ notifyEvent<HWInstructionEvent>(
HWInstructionEvent(HWInstructionEvent::Ready, IR));
- S.notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, UsedRes));
+ notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, UsedRes));
LLVM_DEBUG(dbgs() << "[E] Issued #" << IR << "\n");
}
-static void notifyInstructionDispatch(const InstRef &IR, unsigned Ops,
- const SmallVectorImpl<unsigned> &UsedRegs,
- const Stage &S) {
-
- S.notifyEvent<HWInstructionEvent>(
+void InOrderIssueStage::notifyInstructionDispatched(
+ const InstRef &IR, unsigned Ops, ArrayRef<unsigned> UsedRegs) {
+ notifyEvent<HWInstructionEvent>(
HWInstructionDispatchedEvent(IR, UsedRegs, Ops));
LLVM_DEBUG(dbgs() << "[E] Dispatched #" << IR << "\n");
}
+void InOrderIssueStage::notifyInstructionExecuted(const InstRef &IR) {
+ notifyEvent<HWInstructionEvent>(
+ HWInstructionEvent(HWInstructionEvent::Executed, IR));
+ LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR << " is executed\n");
+}
+
+void InOrderIssueStage::notifyInstructionRetired(const InstRef &IR,
+ ArrayRef<unsigned> FreedRegs) {
+ notifyEvent<HWInstructionEvent>(HWInstructionRetiredEvent(IR, FreedRegs));
+ LLVM_DEBUG(dbgs() << "[E] Retired #" << IR << " \n");
+}
+
llvm::Error InOrderIssueStage::execute(InstRef &IR) {
- if (llvm::Error E = tryIssue(IR, &StallCyclesLeft))
+ if (llvm::Error E = tryIssue(IR))
return E;
- if (StallCyclesLeft) {
- StalledInst = IR;
- }
+ if (SI.isValid())
+ notifyStallEvent();
return llvm::ErrorSuccess();
}
-llvm::Error InOrderIssueStage::tryIssue(InstRef &IR, unsigned *StallCycles) {
+llvm::Error InOrderIssueStage::tryIssue(InstRef &IR) {
Instruction &IS = *IR.getInstruction();
unsigned SourceIndex = IR.getSourceIndex();
const InstrDesc &Desc = IS.getDesc();
- if (!canExecute(IR, StallCycles)) {
- LLVM_DEBUG(dbgs() << "[E] Stalled #" << IR << " for " << *StallCycles
- << " cycles\n");
+ if (!canExecute(IR)) {
+ LLVM_DEBUG(dbgs() << "[N] Stalled #" << SI.getInstruction() << " for "
+ << SI.getCyclesLeft() << " cycles\n");
Bandwidth = 0;
return llvm::ErrorSuccess();
}
@@ -225,18 +181,18 @@ llvm::Error InOrderIssueStage::tryIssue(InstRef &IR, unsigned *StallCycles) {
addRegisterReadWrite(PRF, IS, SourceIndex, STI, UsedRegs);
unsigned NumMicroOps = IS.getNumMicroOps();
- notifyInstructionDispatch(IR, NumMicroOps, UsedRegs, *this);
+ notifyInstructionDispatched(IR, NumMicroOps, UsedRegs);
- SmallVector<std::pair<ResourceRef, ResourceCycles>, 4> UsedResources;
+ SmallVector<ResourceUse, 4> UsedResources;
RM->issueInstruction(Desc, UsedResources);
IS.execute(SourceIndex);
// Replace resource masks with valid resource processor IDs.
- for (std::pair<ResourceRef, ResourceCycles> &Use : UsedResources) {
+ for (ResourceUse &Use : UsedResources) {
uint64_t Mask = Use.first.first;
Use.first.first = RM->resolveResourceMask(Mask);
}
- notifyInstructionIssue(IR, UsedResources, *this);
+ notifyInstructionIssued(IR, UsedResources);
bool ShouldCarryOver = NumMicroOps > Bandwidth;
if (ShouldCarryOver) {
@@ -269,16 +225,14 @@ void InOrderIssueStage::updateIssuedInst() {
IS.cycleEvent();
if (!IS.isExecuted()) {
- LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR
+ LLVM_DEBUG(dbgs() << "[N] Instruction #" << IR
<< " is still executing\n");
++I;
continue;
}
PRF.onInstructionExecuted(&IS);
- notifyEvent<HWInstructionEvent>(
- HWInstructionEvent(HWInstructionEvent::Executed, IR));
- LLVM_DEBUG(dbgs() << "[E] Instruction #" << IR << " is executed\n");
+ notifyInstructionExecuted(IR);
++NumExecuted;
retireInstruction(*I);
@@ -294,18 +248,17 @@ void InOrderIssueStage::updateCarriedOver() {
if (!CarriedOver)
return;
- assert(!StalledInst && "A stalled instruction cannot be carried over.");
+ assert(!SI.isValid() && "A stalled instruction cannot be carried over.");
if (CarryOver > Bandwidth) {
CarryOver -= Bandwidth;
Bandwidth = 0;
LLVM_DEBUG(dbgs() << "[N] Carry over (" << CarryOver << "uops left) #"
- << CarriedOver << " \n");
+ << CarriedOver << " \n");
return;
}
- LLVM_DEBUG(dbgs() << "[N] Carry over (complete) #" << CarriedOver
- << " \n");
+ LLVM_DEBUG(dbgs() << "[N] Carry over (complete) #" << CarriedOver << " \n");
if (CarriedOver.getInstruction()->getDesc().EndGroup)
Bandwidth = 0;
@@ -324,8 +277,33 @@ void InOrderIssueStage::retireInstruction(InstRef &IR) {
for (const WriteState &WS : IS.getDefs())
PRF.removeRegisterWrite(WS, FreedRegs);
- notifyEvent<HWInstructionEvent>(HWInstructionRetiredEvent(IR, FreedRegs));
- LLVM_DEBUG(dbgs() << "[E] Retired #" << IR << " \n");
+ notifyInstructionRetired(IR, FreedRegs);
+}
+
+void InOrderIssueStage::notifyStallEvent() {
+ assert(SI.getCyclesLeft() && "A zero cycles stall?");
+ assert(SI.isValid() && "Invalid stall information found!");
+
+ const InstRef &IR = SI.getInstruction();
+
+ switch (SI.getStallKind()) {
+ default:
+ break;
+ case StallInfo::StallKind::REGISTER_DEPS: {
+ notifyEvent<HWStallEvent>(
+ HWStallEvent(HWStallEvent::RegisterFileStall, IR));
+ notifyEvent<HWPressureEvent>(
+ HWPressureEvent(HWPressureEvent::REGISTER_DEPS, IR));
+ break;
+ }
+ case StallInfo::StallKind::DISPATCH: {
+ notifyEvent<HWStallEvent>(
+ HWStallEvent(HWStallEvent::DispatchGroupStall, IR));
+ notifyEvent<HWPressureEvent>(
+ HWPressureEvent(HWPressureEvent::RESOURCES, IR));
+ break;
+ }
+ }
}
llvm::Error InOrderIssueStage::cycleStart() {
@@ -344,28 +322,34 @@ llvm::Error InOrderIssueStage::cycleStart() {
updateCarriedOver();
// Issue instructions scheduled for this cycle
- if (!StallCyclesLeft && StalledInst) {
- if (llvm::Error E = tryIssue(StalledInst, &StallCyclesLeft))
- return E;
- }
+ if (SI.isValid()) {
+ if (!SI.getCyclesLeft()) {
+ // Make a copy of the reference, and try issue it again.
+ // Do not take the instruction reference because SI.clear() will
+ // invalidate it.
+ InstRef IR = SI.getInstruction();
+ SI.clear();
+
+ if (llvm::Error E = tryIssue(IR))
+ return E;
+ }
- if (!StallCyclesLeft) {
- StalledInst.invalidate();
- assert(NumIssued <= SM.IssueWidth && "Overflow.");
- } else {
- // The instruction is still stalled, cannot issue any new instructions in
- // this cycle.
- Bandwidth = 0;
+ if (SI.getCyclesLeft()) {
+ // The instruction is still stalled, cannot issue any new instructions in
+ // this cycle.
+ notifyStallEvent();
+ Bandwidth = 0;
+ return llvm::ErrorSuccess();
+ }
}
+ assert(NumIssued <= SM.IssueWidth && "Overflow.");
return llvm::ErrorSuccess();
}
llvm::Error InOrderIssueStage::cycleEnd() {
PRF.cycleEnd();
-
- if (StallCyclesLeft > 0)
- --StallCyclesLeft;
+ SI.cycleEnd();
if (LastWriteBackCycle > 0)
--LastWriteBackCycle;
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
index 38a8e2ef9c53c..0fe1cb091fb9d 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
@@ -66,8 +66,6 @@ void PressureTracker::onInstructionExecuted(unsigned IID) { IPI.erase(IID); }
void PressureTracker::handleInstructionIssuedEvent(
const HWInstructionIssuedEvent &Event) {
unsigned IID = Event.IR.getSourceIndex();
- using ResourceRef = HWInstructionIssuedEvent::ResourceRef;
- using ResourceUse = std::pair<ResourceRef, ResourceCycles>;
for (const ResourceUse &Use : Event.UsedResources) {
const ResourceRef &RR = Use.first;
unsigned Index = ProcResID2ResourceUsersIndex[RR.first];
More information about the llvm-commits
mailing list