[llvm-branch-commits] [clang] [compiler-rt] [libcxx] [llvm] [mlir] [CodeGen][NewPM] Port RegAllocGreedy to NPM (PR #119540)
Akshat Oke via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Feb 18 04:05:25 PST 2025
=?utf-8?q?Donát?= Nagy <donat.nagy at ericsson.com>,Matt Arsenault
<Matthew.Arsenault at amd.com>,Christian Sigg <csigg at google.com>,Ramkumar
Ramachandra <ramkumar.ramachandra at codasip.com>,David Green
<david.green at arm.com>,
Andrzej =?utf-8?q?Warzyński?= <andrzej.warzynski at arm.com>,Vyacheslav
Levytskyy <vyacheslav.levytskyy at intel.com>,Steven Cooreman
<steven.cooreman at gmail.com>,Lang Hames <lhames at gmail.com>,Antonio Frighetto
<me at antoniofrighetto.com>,Akshat Oke <Akshat.Oke at amd.com>,Christian Sigg
<csigg at google.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>,Akshat Oke
<Akshat.Oke at amd.com>,Akshat Oke <Akshat.Oke at amd.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/119540 at github.com>
https://github.com/optimisan updated https://github.com/llvm/llvm-project/pull/119540
>From bc4f05d8a8a4f908252aba9444571de4398d4288 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec at users.noreply.github.com>
Date: Tue, 18 Feb 2025 02:08:28 -0800
Subject: [PATCH 01/31] [AMDGPU] Early bail in getFunctionCodeSize for meta
inst. NFC. (#127129)
It does not change the estimate because getInstSizeInBytes() already
returns 0 for meta instructions, but added a test and early bail.
---
llvm/lib/Target/AMDGPU/SIProgramInfo.cpp | 3 +--
llvm/test/CodeGen/AMDGPU/code-size-estimate.mir | 13 +++++++++++++
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
index 5179288084010..1123696509818 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
@@ -215,8 +215,7 @@ uint64_t SIProgramInfo::getFunctionCodeSize(const MachineFunction &MF) {
for (const MachineInstr &MI : MBB) {
// TODO: CodeSize should account for multiple functions.
- // TODO: Should we count size of debug info?
- if (MI.isDebugInstr())
+ if (MI.isMetaInstruction())
continue;
CodeSize += TII->getInstSizeInBytes(MI);
diff --git a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
index 9e46c58b6b5a9..76eaf350301e4 100644
--- a/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
+++ b/llvm/test/CodeGen/AMDGPU/code-size-estimate.mir
@@ -18,3 +18,16 @@ body: |
$vgpr16 = V_MOV_B32_indirect_read undef $vgpr1, implicit $exec, implicit $m0, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
V_MOV_B32_indirect_write undef $vgpr0, undef $vgpr3, implicit $exec, implicit $m0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3(tied-def 4)
...
+
+# CHECK: meta: ; @meta
+# CHECK: ; wave barrier
+# CHECK: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
+# CHECK: ; codeLenInByte = 4
+---
+name: meta
+tracksRegLiveness: true
+body: |
+ bb.0:
+
+ WAVE_BARRIER
+...
>From 7c03865a1ec6de2b734d8cbf75ca2e79ac6d013d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 18 Feb 2025 17:16:50 +0700
Subject: [PATCH 02/31] AMDGPU: Extract lambda used in foldImmediate into a
helper function (#127484)
It was also too permissive for a more general utilty, only return
the original immediate if there is no subregister.
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 59 ++++++++++++++++----------
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 9 ++++
2 files changed, 45 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 4ee5ebd7681b8..07addb38b8711 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -3437,6 +3437,30 @@ void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
}
}
+std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
+ unsigned SubRegIndex) {
+ switch (SubRegIndex) {
+ case AMDGPU::NoSubRegister:
+ return Imm;
+ case AMDGPU::sub0:
+ return Lo_32(Imm);
+ case AMDGPU::sub1:
+ return Hi_32(Imm);
+ case AMDGPU::lo16:
+ return SignExtend64<16>(Imm);
+ case AMDGPU::hi16:
+ return SignExtend64<16>(Imm >> 16);
+ case AMDGPU::sub1_lo16:
+ return SignExtend64<16>(Imm >> 32);
+ case AMDGPU::sub1_hi16:
+ return SignExtend64<16>(Imm >> 48);
+ default:
+ return std::nullopt;
+ }
+
+ llvm_unreachable("covered subregister switch");
+}
+
bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
Register Reg, MachineRegisterInfo *MRI) const {
if (!MRI->hasOneNonDBGUse(Reg))
@@ -3446,25 +3470,6 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
if (!getConstValDefinedInReg(DefMI, Reg, Imm))
return false;
- auto getImmFor = [=](const MachineOperand &UseOp) -> int64_t {
- switch (UseOp.getSubReg()) {
- default:
- return Imm;
- case AMDGPU::sub0:
- return Lo_32(Imm);
- case AMDGPU::sub1:
- return Hi_32(Imm);
- case AMDGPU::lo16:
- return SignExtend64<16>(Imm);
- case AMDGPU::hi16:
- return SignExtend64<16>(Imm >> 16);
- case AMDGPU::sub1_lo16:
- return SignExtend64<16>(Imm >> 32);
- case AMDGPU::sub1_hi16:
- return SignExtend64<16>(Imm >> 48);
- }
- };
-
assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
unsigned Opc = UseMI.getOpcode();
@@ -3480,7 +3485,11 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
: AMDGPU::V_MOV_B32_e32
: Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO
: AMDGPU::S_MOV_B32;
- APInt Imm(Is64Bit ? 64 : 32, getImmFor(UseMI.getOperand(1)),
+
+ std::optional<int64_t> SubRegImm =
+ extractSubregFromImm(Imm, UseMI.getOperand(1).getSubReg());
+
+ APInt Imm(Is64Bit ? 64 : 32, *SubRegImm,
/*isSigned=*/true, /*implicitTrunc=*/true);
if (RI.isAGPR(*MRI, DstReg)) {
@@ -3591,7 +3600,8 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
if (NewOpc == AMDGPU::V_FMAMK_F16_fake16)
return false;
- const int64_t Imm = getImmFor(RegSrc == Src1 ? *Src0 : *Src1);
+ const std::optional<int64_t> SubRegImm = extractSubregFromImm(
+ Imm, RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
// FIXME: This would be a lot easier if we could return a new instruction
// instead of having to modify in place.
@@ -3608,7 +3618,7 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
UseMI.untieRegOperand(
AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
- Src1->ChangeToImmediate(Imm);
+ Src1->ChangeToImmediate(*SubRegImm);
removeModOperands(UseMI);
UseMI.setDesc(get(NewOpc));
@@ -3679,8 +3689,11 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
UseMI.untieRegOperand(
AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
+ const std::optional<int64_t> SubRegImm =
+ extractSubregFromImm(Imm, Src2->getSubReg());
+
// ChangingToImmediate adds Src2 back to the instruction.
- Src2->ChangeToImmediate(getImmFor(*Src2));
+ Src2->ChangeToImmediate(*SubRegImm);
// These come before src2.
removeModOperands(UseMI);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index ddd15e1766f70..79ecc2a657ed0 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -401,6 +401,15 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
void removeModOperands(MachineInstr &MI) const;
+ /// Return the extracted immediate value in a subregister use from a constant
+ /// materialized in a super register.
+ ///
+ /// e.g. %imm = S_MOV_B64 K[0:63]
+ /// USE %imm.sub1
+ /// This will return K[32:63]
+ static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
+ unsigned SubRegIndex);
+
bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
MachineRegisterInfo *MRI) const final;
>From 1c8add1ec70d8d730572029ac11a70f4dfac8ed5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy at ericsson.com>
Date: Tue, 18 Feb 2025 11:19:43 +0100
Subject: [PATCH 03/31] [analyzer] Add hack in ArrayBound to cover up missing
casts (#127117)
Currently there are many casts that are not modeled (i.e. ignored) by
the analyzer, which can cause paradox states (e.g. negative value stored
in `unsigned` variable) and false positive reports from various
checkers, e.g. from `security.ArrayBound`.
Unfortunately this issue is deeply rooted in the architectural
limitations of the analyzer (if we started to model the casts, it would
break other things). For details see the umbrella ticket
https://github.com/llvm/llvm-project/issues/39492
This commit adds an ugly hack in `security.ArrayBound` to silence most
of the false positives caused by this shortcoming of the engine.
Fixes #126884
---
.../Checkers/ArrayBoundChecker.cpp | 99 ++++++++++++++-----
clang/test/Analysis/out-of-bounds.c | 22 +++--
2 files changed, 87 insertions(+), 34 deletions(-)
diff --git a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
index f56e9192d1d66..954b4763034e7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
@@ -34,24 +34,37 @@ using namespace taint;
using llvm::formatv;
namespace {
-/// If `E` is a "clean" array subscript expression, return the type of the
-/// accessed element. If the base of the subscript expression is modified by
-/// pointer arithmetic (and not the beginning of a "full" memory region), this
-/// always returns nullopt because that's the right (or the least bad) thing to
-/// do for the diagnostic output that's relying on this.
-static std::optional<QualType> determineElementType(const Expr *E,
- const CheckerContext &C) {
+/// If `E` is an array subscript expression with a base that is "clean" (= not
+/// modified by pointer arithmetic = the beginning of a memory region), return
+/// it as a pointer to ArraySubscriptExpr; otherwise return nullptr.
+/// This helper function is used by two separate heuristics that are only valid
+/// in these "clean" cases.
+static const ArraySubscriptExpr *
+getAsCleanArraySubscriptExpr(const Expr *E, const CheckerContext &C) {
const auto *ASE = dyn_cast<ArraySubscriptExpr>(E);
if (!ASE)
- return std::nullopt;
+ return nullptr;
const MemRegion *SubscriptBaseReg = C.getSVal(ASE->getBase()).getAsRegion();
if (!SubscriptBaseReg)
- return std::nullopt;
+ return nullptr;
// The base of the subscript expression is affected by pointer arithmetics,
- // so we want to report byte offsets instead of indices.
+ // so we want to report byte offsets instead of indices and we don't want to
+ // activate the "index is unsigned -> cannot be negative" shortcut.
if (isa<ElementRegion>(SubscriptBaseReg->StripCasts()))
+ return nullptr;
+
+ return ASE;
+}
+
+/// If `E` is a "clean" array subscript expression, return the type of the
+/// accessed element; otherwise return std::nullopt because that's the best (or
+/// least bad) option for the diagnostic generation that relies on this.
+static std::optional<QualType> determineElementType(const Expr *E,
+ const CheckerContext &C) {
+ const auto *ASE = getAsCleanArraySubscriptExpr(E, C);
+ if (!ASE)
return std::nullopt;
return ASE->getType();
@@ -140,7 +153,9 @@ class ArrayBoundChecker : public Checker<check::PostStmt<ArraySubscriptExpr>,
ProgramStateRef ErrorState, NonLoc Val,
bool MarkTaint);
- static bool isFromCtypeMacro(const Stmt *S, ASTContext &AC);
+ static bool isFromCtypeMacro(const Expr *E, ASTContext &AC);
+
+ static bool isOffsetObviouslyNonnegative(const Expr *E, CheckerContext &C);
static bool isIdiomaticPastTheEndPtr(const Expr *E, ProgramStateRef State,
NonLoc Offset, NonLoc Limit,
@@ -587,20 +602,48 @@ void ArrayBoundChecker::performCheck(const Expr *E, CheckerContext &C) const {
State, ByteOffset, SVB.makeZeroArrayIndex(), SVB);
if (PrecedesLowerBound) {
- // The offset may be invalid (negative)...
- if (!WithinLowerBound) {
- // ...and it cannot be valid (>= 0), so report an error.
- Messages Msgs = getPrecedesMsgs(Reg, ByteOffset);
- reportOOB(C, PrecedesLowerBound, Msgs, ByteOffset, std::nullopt);
- return;
+ // The analyzer thinks that the offset may be invalid (negative)...
+
+ if (isOffsetObviouslyNonnegative(E, C)) {
+ // ...but the offset is obviously non-negative (clear array subscript
+ // with an unsigned index), so we're in a buggy situation.
+
+ // TODO: Currently the analyzer ignores many casts (e.g. signed ->
+ // unsigned casts), so it can easily reach states where it will load a
+ // signed (and negative) value from an unsigned variable. This sanity
+ // check is a duct tape "solution" that silences most of the ugly false
+ // positives that are caused by this buggy behavior. Note that this is
+ // not a complete solution: this cannot silence reports where pointer
+ // arithmetic complicates the picture and cannot ensure modeling of the
+ // "unsigned index is positive with highest bit set" cases which are
+ // "usurped" by the nonsense "unsigned index is negative" case.
+ // For more information about this topic, see the umbrella ticket
+ // https://github.com/llvm/llvm-project/issues/39492
+ // TODO: Remove this hack once 'SymbolCast's are modeled properly.
+
+ if (!WithinLowerBound) {
+ // The state is completely nonsense -- let's just sink it!
+ C.addSink();
+ return;
+ }
+ // Otherwise continue on the 'WithinLowerBound' branch where the
+ // unsigned index _is_ non-negative. Don't mention this assumption as a
+ // note tag, because it would just confuse the users!
+ } else {
+ if (!WithinLowerBound) {
+ // ...and it cannot be valid (>= 0), so report an error.
+ Messages Msgs = getPrecedesMsgs(Reg, ByteOffset);
+ reportOOB(C, PrecedesLowerBound, Msgs, ByteOffset, std::nullopt);
+ return;
+ }
+ // ...but it can be valid as well, so the checker will (optimistically)
+ // assume that it's valid and mention this in the note tag.
+ SUR.recordNonNegativeAssumption();
}
- // ...but it can be valid as well, so the checker will (optimistically)
- // assume that it's valid and mention this in the note tag.
- SUR.recordNonNegativeAssumption();
}
// Actually update the state. The "if" only fails in the extremely unlikely
- // case when compareValueToThreshold returns {nullptr, nullptr} becasue
+ // case when compareValueToThreshold returns {nullptr, nullptr} because
// evalBinOpNN fails to evaluate the less-than operator.
if (WithinLowerBound)
State = WithinLowerBound;
@@ -660,7 +703,7 @@ void ArrayBoundChecker::performCheck(const Expr *E, CheckerContext &C) const {
}
// Actually update the state. The "if" only fails in the extremely unlikely
- // case when compareValueToThreshold returns {nullptr, nullptr} becasue
+ // case when compareValueToThreshold returns {nullptr, nullptr} because
// evalBinOpNN fails to evaluate the less-than operator.
if (WithinUpperBound)
State = WithinUpperBound;
@@ -725,8 +768,8 @@ void ArrayBoundChecker::reportOOB(CheckerContext &C, ProgramStateRef ErrorState,
C.emitReport(std::move(BR));
}
-bool ArrayBoundChecker::isFromCtypeMacro(const Stmt *S, ASTContext &ACtx) {
- SourceLocation Loc = S->getBeginLoc();
+bool ArrayBoundChecker::isFromCtypeMacro(const Expr *E, ASTContext &ACtx) {
+ SourceLocation Loc = E->getBeginLoc();
if (!Loc.isMacroID())
return false;
@@ -744,6 +787,14 @@ bool ArrayBoundChecker::isFromCtypeMacro(const Stmt *S, ASTContext &ACtx) {
(MacroName == "isupper") || (MacroName == "isxdigit"));
}
+bool ArrayBoundChecker::isOffsetObviouslyNonnegative(const Expr *E,
+ CheckerContext &C) {
+ const ArraySubscriptExpr *ASE = getAsCleanArraySubscriptExpr(E, C);
+ if (!ASE)
+ return false;
+ return ASE->getIdx()->getType()->isUnsignedIntegerOrEnumerationType();
+}
+
bool ArrayBoundChecker::isInAddressOf(const Stmt *S, ASTContext &ACtx) {
ParentMapContext &ParentCtx = ACtx.getParentMapContext();
do {
diff --git a/clang/test/Analysis/out-of-bounds.c b/clang/test/Analysis/out-of-bounds.c
index 7a094b8fdc840..7d6cb4ecf1b24 100644
--- a/clang/test/Analysis/out-of-bounds.c
+++ b/clang/test/Analysis/out-of-bounds.c
@@ -188,29 +188,31 @@ int test_cast_to_unsigned(signed char x) {
if (x >= 0)
return x;
// FIXME: Here the analyzer ignores the signed -> unsigned cast, and manages to
- // load a negative value from an unsigned variable. This causes an underflow
- // report, which is an ugly false positive.
+ // load a negative value from an unsigned variable.
// The underlying issue is tracked by Github ticket #39492.
clang_analyzer_value(y); // expected-warning {{8s:{ [-128, -1] } }}
- return table[y]; // expected-warning {{Out of bound access to memory preceding}}
+ // However, a hack in the ArrayBound checker suppresses the false positive
+ // underflow report that would be generated here.
+ return table[y]; // no-warning
}
int test_cast_to_unsigned_overflow(signed char x) {
unsigned char y = x;
if (x >= 0)
return x;
- // A variant of 'test_cast_to_unsigned' where the correct behavior would be
- // an overflow report (because the negative values are cast to `unsigned
- // char` values that are too large).
- // FIXME: See comment in 'test_cast_to_unsigned'.
+ // FIXME: As in 'test_cast_to_unsigned', the analyzer thinks that this
+ // unsigned variable contains a negative value.
clang_analyzer_value(y); // expected-warning {{8s:{ [-128, -1] } }}
- return small_table[y]; // expected-warning {{Out of bound access to memory preceding}}
+ // FIXME: The following subscript expression should produce an overflow
+ // report (because negative signed char corresponds to unsigned char >= 128);
+ // but the hack in ArrayBound just silences reports and cannot "restore" the
+ // real execution paths.
+ return small_table[y]; // no-warning
}
int test_negative_offset_with_unsigned_idx(void) {
// An example where the subscript operator uses an unsigned index, but the
- // underflow report is still justified. (We should try to keep this if we
- // silence false positives like the one in 'test_cast_to_unsigned'.)
+ // underflow report is still justified.
int *p = table - 10;
unsigned idx = 2u;
return p[idx]; // expected-warning {{Out of bound access to memory preceding}}
>From cd10c01767f1d06748c1d4502cc4384a18c06115 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Tue, 18 Feb 2025 17:19:53 +0700
Subject: [PATCH 04/31] AMDGPU: Handle subregister uses in SIFoldOperands
constant folding (#127485)
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 57 +++++++++++--------
.../AMDGPU/constant-fold-imm-immreg.mir | 34 +++++++++++
2 files changed, 67 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 7c08a21dea3b8..ab396929162d0 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -123,7 +123,7 @@ class SIFoldOperandsImpl {
SmallVectorImpl<FoldCandidate> &FoldList,
SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
- MachineOperand *getImmOrMaterializedImm(MachineOperand &Op) const;
+ std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
bool tryConstantFoldOp(MachineInstr *MI) const;
bool tryFoldCndMask(MachineInstr &MI) const;
bool tryFoldZeroHighBits(MachineInstr &MI) const;
@@ -1296,21 +1296,22 @@ static void mutateCopyOp(MachineInstr &MI, const MCInstrDesc &NewDesc) {
MI.removeOperand(I);
}
-MachineOperand *
+std::optional<int64_t>
SIFoldOperandsImpl::getImmOrMaterializedImm(MachineOperand &Op) const {
- // If this has a subregister, it obviously is a register source.
- if (!Op.isReg() || Op.getSubReg() != AMDGPU::NoSubRegister ||
- !Op.getReg().isVirtual())
- return &Op;
+ if (Op.isImm())
+ return Op.getImm();
- MachineInstr *Def = MRI->getVRegDef(Op.getReg());
+ if (!Op.isReg() || !Op.getReg().isVirtual())
+ return std::nullopt;
+
+ const MachineInstr *Def = MRI->getVRegDef(Op.getReg());
if (Def && Def->isMoveImmediate()) {
- MachineOperand &ImmSrc = Def->getOperand(1);
+ const MachineOperand &ImmSrc = Def->getOperand(1);
if (ImmSrc.isImm())
- return &ImmSrc;
+ return TII->extractSubregFromImm(ImmSrc.getImm(), Op.getSubReg());
}
- return &Op;
+ return std::nullopt;
}
// Try to simplify operations with a constant that may appear after instruction
@@ -1325,12 +1326,14 @@ bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
if (Src0Idx == -1)
return false;
- MachineOperand *Src0 = getImmOrMaterializedImm(MI->getOperand(Src0Idx));
+
+ MachineOperand *Src0 = &MI->getOperand(Src0Idx);
+ std::optional<int64_t> Src0Imm = getImmOrMaterializedImm(*Src0);
if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
Opc == AMDGPU::S_NOT_B32) &&
- Src0->isImm()) {
- MI->getOperand(1).ChangeToImmediate(~Src0->getImm());
+ Src0Imm) {
+ MI->getOperand(1).ChangeToImmediate(~*Src0Imm);
mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32)));
return true;
}
@@ -1338,17 +1341,19 @@ bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
if (Src1Idx == -1)
return false;
- MachineOperand *Src1 = getImmOrMaterializedImm(MI->getOperand(Src1Idx));
- if (!Src0->isImm() && !Src1->isImm())
+ MachineOperand *Src1 = &MI->getOperand(Src1Idx);
+ std::optional<int64_t> Src1Imm = getImmOrMaterializedImm(*Src1);
+
+ if (!Src0Imm && !Src1Imm)
return false;
// and k0, k1 -> v_mov_b32 (k0 & k1)
// or k0, k1 -> v_mov_b32 (k0 | k1)
// xor k0, k1 -> v_mov_b32 (k0 ^ k1)
- if (Src0->isImm() && Src1->isImm()) {
+ if (Src0Imm && Src1Imm) {
int32_t NewImm;
- if (!evalBinaryInstruction(Opc, NewImm, Src0->getImm(), Src1->getImm()))
+ if (!evalBinaryInstruction(Opc, NewImm, *Src0Imm, *Src1Imm))
return false;
bool IsSGPR = TRI->isSGPRReg(*MRI, MI->getOperand(0).getReg());
@@ -1364,12 +1369,13 @@ bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
if (!MI->isCommutable())
return false;
- if (Src0->isImm() && !Src1->isImm()) {
+ if (Src0Imm && !Src1Imm) {
std::swap(Src0, Src1);
std::swap(Src0Idx, Src1Idx);
+ std::swap(Src0Imm, Src1Imm);
}
- int32_t Src1Val = static_cast<int32_t>(Src1->getImm());
+ int32_t Src1Val = static_cast<int32_t>(*Src1Imm);
if (Opc == AMDGPU::V_OR_B32_e64 ||
Opc == AMDGPU::V_OR_B32_e32 ||
Opc == AMDGPU::S_OR_B32) {
@@ -1426,9 +1432,12 @@ bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
if (!Src1->isIdenticalTo(*Src0)) {
- auto *Src0Imm = getImmOrMaterializedImm(*Src0);
- auto *Src1Imm = getImmOrMaterializedImm(*Src1);
- if (!Src1Imm->isIdenticalTo(*Src0Imm))
+ std::optional<int64_t> Src1Imm = getImmOrMaterializedImm(*Src1);
+ if (!Src1Imm)
+ return false;
+
+ std::optional<int64_t> Src0Imm = getImmOrMaterializedImm(*Src0);
+ if (!Src0Imm || *Src0Imm != *Src1Imm)
return false;
}
@@ -1461,8 +1470,8 @@ bool SIFoldOperandsImpl::tryFoldZeroHighBits(MachineInstr &MI) const {
MI.getOpcode() != AMDGPU::V_AND_B32_e32)
return false;
- MachineOperand *Src0 = getImmOrMaterializedImm(MI.getOperand(1));
- if (!Src0->isImm() || Src0->getImm() != 0xffff)
+ std::optional<int64_t> Src0Imm = getImmOrMaterializedImm(MI.getOperand(1));
+ if (!Src0Imm || *Src0Imm != 0xffff)
return false;
Register Src1 = MI.getOperand(2).getReg();
diff --git a/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir b/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
index 39b5076ebe5ac..807eaf2160b3c 100644
--- a/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
@@ -927,3 +927,37 @@ body: |
S_ENDPGM 0, implicit %3
...
+
+---
+name: constant_s_xor_b32_uses_subreg
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GCN-LABEL: name: constant_s_xor_b32_uses_subreg
+ ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 47
+ ; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; GCN-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]]
+ %0:sreg_64 = S_MOV_B64 32
+ %1:sreg_64 = S_MOV_B64 15
+ %2:sgpr_32 = S_XOR_B32 %0.sub0, %1.sub0, implicit-def dead $scc
+ %3:sgpr_32 = S_XOR_B32 %0.sub1, %1.sub1, implicit-def dead $scc
+ S_ENDPGM 0, implicit %2, implicit %3
+
+...
+
+---
+name: constant_v_or_b32_uses_subreg
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; GCN-LABEL: name: constant_v_or_b32_uses_subreg
+ ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 268435455, implicit $exec
+ ; GCN-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+ ; GCN-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]]
+ %0:vreg_64 = V_MOV_B64_PSEUDO 18446744069683019775, implicit $exec
+ %1:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ %2:vgpr_32 = V_OR_B32_e32 %0.sub0, %1.sub0, implicit $exec
+ %3:vgpr_32 = V_OR_B32_e32 %0.sub1, %1.sub1, implicit $exec
+ S_ENDPGM 0, implicit %2, implicit %3
+
+...
>From ef218317d7e30eb32a8e4e1a68d3fe95ca3b8402 Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg at google.com>
Date: Tue, 18 Feb 2025 11:24:15 +0100
Subject: [PATCH 05/31] [bolt][bazel] Port
https://github.com/llvm/llvm-project/commit/e235fcb582eec5f58c905b66f96d0732d17b875e.
---
utils/bazel/configure.bzl | 13 +++++++++++--
.../bazel/llvm-project-overlay/bolt/BUILD.bazel | 17 +++++++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/utils/bazel/configure.bzl b/utils/bazel/configure.bzl
index c5da28845eccf..fcc9fc7ecc483 100644
--- a/utils/bazel/configure.bzl
+++ b/utils/bazel/configure.bzl
@@ -172,10 +172,19 @@ def _llvm_configure_impl(repository_ctx):
)
# Create a starlark file with the requested LLVM targets.
- targets = repository_ctx.attr.targets
+ llvm_targets = repository_ctx.attr.targets
repository_ctx.file(
"llvm/targets.bzl",
- content = "llvm_targets = " + str(targets),
+ content = "llvm_targets = " + str(llvm_targets),
+ executable = False,
+ )
+
+ # Create a starlark file with the requested BOLT targets.
+ bolt_targets = ["AArch64","X86","RISCV"] # Supported targets.
+ bolt_targets = [t for t in llvm_targets if t in bolt_targets]
+ repository_ctx.file(
+ "bolt/targets.bzl",
+ content = "bolt_targets = " + str(bolt_targets),
executable = False,
)
diff --git a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
index 187938783a550..a9a7cc59575a3 100644
--- a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
+load(":targets.bzl", "bolt_targets")
package(
default_visibility = ["//visibility:public"],
@@ -16,6 +17,20 @@ genrule(
cmd = "echo '#undef BOLT_REVISION' >> $@\n",
)
+expand_template(
+ name = "target_config_def_gen",
+ out = "include/bolt/Core/TargetConfig.def",
+ substitutions = {"@BOLT_ENUM_TARGETS@": "\n".join(
+ ["BOLT_TARGET({})".format(target) for target in bolt_targets],
+ )},
+ template = "include/bolt/Core/TargetConfig.def.in",
+)
+
+cc_library(
+ name = "TargetConfig",
+ textual_hdrs = [":target_config_def_gen"],
+)
+
cc_binary(
name = "llvm-bolt-heatmap",
srcs = glob([
@@ -24,6 +39,7 @@ cc_binary(
deps = [
":Profile",
":Rewrite",
+ ":TargetConfig",
":Utils",
"//llvm:AllTargetsAsmParsers",
"//llvm:AllTargetsDisassemblers",
@@ -54,6 +70,7 @@ cc_binary(
":Profile",
":Rewrite",
":RuntimeLibs",
+ ":TargetConfig",
":TargetAArch64",
":TargetX86",
":Utils",
>From 6646b65082e8bc5f485c744a9c15344011c6aede Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Tue, 18 Feb 2025 10:25:47 +0000
Subject: [PATCH 06/31] [LAA] Rework and rename stripGetElementPtr (#125315)
The stripGetElementPtr function is mysteriously named, and calls into
another mysterious getGEPInductionOperand which does something
complicated with GEP indices. The real purpose of the badly-named
stripGetElementPtr function is to get a loop-variant GEP index, if there
is one. The getGEPInductionOperand is totally redundant, as stripping
off zeros from the end of GEP indices has no effect on computing the
loop-variant GEP index, as constant zeros are always loop-invariant.
Moreover, the GEP induction operand is simply the first non-zero index
from the end, which stripGetElementPtr returns when it finds that any of
the GEP indices are loop-variant: this is a completely unrelated value
to the GEP index that is loop-variant. The implicit assumption here is
that there is only ever one loop-variant index, and it is the first
non-zero one from the end.
The logic is unnecessarily complicated for what stripGetElementPtr wants
to achieve, and the header comments are confusing as well. Strip
getGEPInductionOperand, rework and rename stripGetElementPtr.
---
llvm/lib/Analysis/LoopAccessAnalysis.cpp | 61 ++++++-------------
.../LoopAccessAnalysis/symbolic-stride.ll | 47 ++++++++++++++
2 files changed, 64 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 90db89f745e89..5a22ac8abc3fc 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -42,13 +42,12 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
-#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PassManager.h"
-#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
@@ -66,7 +65,6 @@
#include <vector>
using namespace llvm;
-using namespace llvm::PatternMatch;
#define DEBUG_TYPE "loop-accesses"
@@ -2815,50 +2813,25 @@ bool LoopAccessInfo::isInvariant(Value *V) const {
return SE->isLoopInvariant(S, TheLoop);
}
-/// Find the operand of the GEP that should be checked for consecutive
-/// stores. This ignores trailing indices that have no effect on the final
-/// pointer.
-static unsigned getGEPInductionOperand(const GetElementPtrInst *Gep) {
- const DataLayout &DL = Gep->getDataLayout();
- unsigned LastOperand = Gep->getNumOperands() - 1;
- TypeSize GEPAllocSize = DL.getTypeAllocSize(Gep->getResultElementType());
-
- // Walk backwards and try to peel off zeros.
- while (LastOperand > 1 && match(Gep->getOperand(LastOperand), m_Zero())) {
- // Find the type we're currently indexing into.
- gep_type_iterator GEPTI = gep_type_begin(Gep);
- std::advance(GEPTI, LastOperand - 2);
-
- // If it's a type with the same allocation size as the result of the GEP we
- // can peel off the zero index.
- TypeSize ElemSize = GEPTI.isStruct()
- ? DL.getTypeAllocSize(GEPTI.getIndexedType())
- : GEPTI.getSequentialElementStride(DL);
- if (ElemSize != GEPAllocSize)
- break;
- --LastOperand;
- }
-
- return LastOperand;
-}
-
-/// If the argument is a GEP, then returns the operand identified by
-/// getGEPInductionOperand. However, if there is some other non-loop-invariant
-/// operand, it returns that instead.
-static Value *stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp) {
+/// If \p Ptr is a GEP, which has a loop-variant operand, return that operand.
+/// Otherwise, return \p Ptr.
+static Value *getLoopVariantGEPOperand(Value *Ptr, ScalarEvolution *SE,
+ Loop *Lp) {
auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
if (!GEP)
return Ptr;
- unsigned InductionOperand = getGEPInductionOperand(GEP);
-
- // Check that all of the gep indices are uniform except for our induction
- // operand.
- for (unsigned I = 0, E = GEP->getNumOperands(); I != E; ++I)
- if (I != InductionOperand &&
- !SE->isLoopInvariant(SE->getSCEV(GEP->getOperand(I)), Lp))
- return Ptr;
- return GEP->getOperand(InductionOperand);
+ Value *V = Ptr;
+ for (const Use &U : GEP->operands()) {
+ if (!SE->isLoopInvariant(SE->getSCEV(U), Lp)) {
+ if (V == Ptr)
+ V = U;
+ else
+ // There must be exactly one loop-variant operand.
+ return Ptr;
+ }
+ }
+ return V;
}
/// Get the stride of a pointer access in a loop. Looks for symbolic
@@ -2873,7 +2846,7 @@ static const SCEV *getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *L
// pointer, otherwise, we are analyzing the index.
Value *OrigPtr = Ptr;
- Ptr = stripGetElementPtr(Ptr, SE, Lp);
+ Ptr = getLoopVariantGEPOperand(Ptr, SE, Lp);
const SCEV *V = SE->getSCEV(Ptr);
if (Ptr != OrigPtr)
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
index 525995156481c..8603417081067 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/symbolic-stride.ll
@@ -140,6 +140,53 @@ exit:
ret void
}
+; Test with multiple GEP indices
+define void @single_stride_array(ptr noalias %A, ptr noalias %B, i64 %N, i64 %stride) {
+; CHECK-LABEL: 'single_stride_array'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT: Backward loop carried data dependence.
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Backward:
+; CHECK-NEXT: %load = load [2 x i32], ptr %gep.A, align 4 ->
+; CHECK-NEXT: store [2 x i32] %ins, ptr %gep.A.next, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-NEXT: Equal predicate: %stride == 1
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+; CHECK-NEXT: [PSE] %gep.A = getelementptr inbounds [2 x i32], ptr %A, i64 %mul, i64 1:
+; CHECK-NEXT: {(4 + %A),+,(8 * %stride)}<%loop>
+; CHECK-NEXT: --> {(4 + %A),+,8}<%loop>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %mul = mul i64 %iv, %stride
+ %gep.A = getelementptr inbounds [2 x i32], ptr %A, i64 %mul, i64 1
+ %load = load [2 x i32], ptr %gep.A, align 4
+ %gep.B = getelementptr inbounds [2 x i32], ptr %B, i64 %iv
+ %load_1 = load [2 x i32], ptr %gep.B, align 4
+ %v1 = extractvalue [2 x i32] %load, 0
+ %v2 = extractvalue [2 x i32] %load_1, 0
+ %add = add i32 %v1, %v2
+ %ins = insertvalue [2 x i32] poison, i32 %add, 0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %gep.A.next = getelementptr inbounds [2 x i32], ptr %A, i64 %iv.next
+ store [2 x i32] %ins, ptr %gep.A.next, align 4
+ %exitcond = icmp eq i64 %iv.next, %N
+ br i1 %exitcond, label %exit, label %loop
+
+exit:
+ ret void
+}
+
define void @single_stride_castexpr(i32 %offset, ptr %src, ptr %dst, i1 %cond) {
; CHECK-LABEL: 'single_stride_castexpr'
; CHECK-NEXT: inner.loop:
>From c71f9141a970b6f6d46d27d7c26c7747dd525275 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 18 Feb 2025 10:48:50 +0000
Subject: [PATCH 07/31] [AArch64] Add a phase-ordering test for dividing
vscale. NFC
See #126411 / #127055, the test isn't expected to fold in a single instcombine
iteration, needing instcombine->cse->instcombine.
---
.../sve-intrinsic-opts-counting-elems.ll | 17 +++++++++++++++++
.../Transforms/PhaseOrdering/AArch64/vscale.ll | 15 +++++++++++++++
2 files changed, 32 insertions(+)
create mode 100644 llvm/test/Transforms/PhaseOrdering/AArch64/vscale.ll
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll
index 4e7e9eeb7250b..46ca99f4bb27b 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll
@@ -240,6 +240,23 @@ define i64 @cntd_all() {
}
+define i64 @udiv() vscale_range(1, 16) {
+; CHECK-LABEL: @udiv(
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[A:%.*]] = shl nuw nsw i64 [[TMP1]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[B:%.*]] = shl nuw nsw i64 [[TMP2]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = call range(i64 2, 65) i64 @llvm.cttz.i64(i64 [[B]], i1 true)
+; CHECK-NEXT: [[C1:%.*]] = lshr i64 [[A]], [[TMP3]]
+; CHECK-NEXT: ret i64 [[C1]]
+;
+ %a = call i64 @llvm.aarch64.sve.cntb(i32 31)
+ %b = call i64 @llvm.aarch64.sve.cntw(i32 31)
+ %c = udiv i64 %a, %b
+ ret i64 %c
+}
+
+
declare i64 @llvm.aarch64.sve.cntb(i32 %pattern)
declare i64 @llvm.aarch64.sve.cnth(i32 %pattern)
declare i64 @llvm.aarch64.sve.cntw(i32 %pattern)
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/vscale.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/vscale.ll
new file mode 100644
index 0000000000000..7aa50ddf61468
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/vscale.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes="default<O1>" -mattr=+sve -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+define i64 @udiv() vscale_range(1, 16) {
+; CHECK-LABEL: @udiv(
+; CHECK-NEXT: ret i64 4
+;
+ %a = call i64 @llvm.aarch64.sve.cntb(i32 31)
+ %b = call i64 @llvm.aarch64.sve.cntw(i32 31)
+ %c = udiv i64 %a, %b
+ ret i64 %c
+}
>From 41be5bbbdba2939a5fdb82c968c102f993edc4d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski at arm.com>
Date: Tue, 18 Feb 2025 10:50:21 +0000
Subject: [PATCH 08/31] [mlir][vector] Update tests for xfer permutation
lowering (3/N) (#127320)
* Remove `vector.create_mask` from tests. Instead, pass masks as
arguments. This simplifies the tests without sacrificing test
coverage.
* Update `@xfer_read_minor_identity_tranposed_with_mask_scalable` to use
similar shapes as other tests and to avoid using test Ops (e.g.
`@test.some_use`). This improves consistency between tests.
* Fix some comment typos.
---
.../vector-transfer-permutation-lowering.mlir | 76 +++++++++----------
1 file changed, 38 insertions(+), 38 deletions(-)
diff --git a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
index 6705905633e0f..dfc79a19e6cc6 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir
@@ -1,5 +1,8 @@
// RUN: mlir-opt %s --transform-interpreter --split-input-file | FileCheck %s
+// TODO: Review the usage of `in_bounds` and remove where not affecting the
+// generated output.
+
/// CHECK: #[[$MAP:.*]] = affine_map<(d0, d1, d2, d3) -> (d1, 0, d3)>
///----------------------------------------------------------------------------------------
@@ -106,8 +109,8 @@ func.func @xfer_write_minor_identity_transposed_map_masked(
/// (neither a minor identity nor transposed minor identity map)
/// OUT 1: vector.broadcast + vector.transfer_write
/// (transposed minor identity)
-/// OUT 2: vector.transfer_write -> vector.broadcast + vector.transpose + vector.transfer_write
-/// (minor identity)
+/// OUT 2: vector.transfer_write -> vector.broadcast + vector.transpose
+/// + vector.transfer_write (minor identity)
///----------------------------------------------------------------------------------------
// CHECK-LABEL: func.func @xfer_write_non_minor_identity(
@@ -233,16 +236,16 @@ func.func @xfer_write_non_minor_identity_masked_scalable(
// CHECK-LABEL: func @xfer_write_non_minor_identity_masked_2
// CHECK-SAME: %[[DEST:.*]]: tensor<?x?x?x?xf32>
// CHECK-SAME: %[[VEC:.*]]: vector<14x8x16xf32>
-// CHECK-SAME: %[[DIM:.*]]: index, %[[IDX:.*]]: index) -> tensor<?x?x?x?xf32>
+// CHECK-SAME: %[[MASK:.*]]: vector<14x8x16xi1>
+// CHECK-SAME: %[[DIM:.*]]: index
// CHECK-NOT: vector.broadcast
-// CHECK: vector.mask %0 { vector.transfer_write %[[VEC]], %[[DEST]]{{.*}} : vector<14x8x16xf32>, tensor<?x?x?x?xf32> } : vector<14x8x16xi1> -> tensor<?x?x?x?xf32>
+// CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[VEC]], %[[DEST]]{{.*}} : vector<14x8x16xf32>, tensor<?x?x?x?xf32> } : vector<14x8x16xi1> -> tensor<?x?x?x?xf32>
func.func @xfer_write_non_minor_identity_masked_2(
%dest : tensor<?x?x?x?xf32>,
%vec : vector<14x8x16xf32>,
- %dim : index,
+ %mask: vector<14x8x16xi1>,
%idx: index) -> tensor<?x?x?x?xf32> {
- %mask = vector.create_mask %dim, %dim, %dim : vector<14x8x16xi1>
%res = vector.mask %mask {
vector.transfer_write %vec, %dest[%idx, %idx, %idx, %idx] {
in_bounds = [false, false, true],
@@ -259,29 +262,27 @@ func.func @xfer_write_non_minor_identity_masked_2(
///
/// IN: vector.transfer_read
/// (_transposed_ minor identity permutation map, with 0 or more broadcast dims)
-/// OUT: vector.transpose + vector.transfer_write
+/// OUT: vector.transfer_read + vector.broadcast + vector.transpose
/// (minor identity permutation map with 0 or more leading broadcast dims)
///----------------------------------------------------------------------------------------
/// TODO: Inner broadcast dim - see also the block at the bottom of this file
-// CHECK-LABEL: func.func @xfer_read_minor_identity_tranposed_with_mask
+// CHECK-LABEL: func.func @xfer_read_minor_identity_transposed_with_mask
// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,
-// CHECK-SAME: %[[DIM_1:.*]]: index, %[[DIM_2:.*]]: index, %[[IDX:.*]]: index) -> vector<8x4x2xf32> {
+// CHECK-SAME: %[[MASK:.*]]: vector<2x4xi1>
+// CHECK-SAME: %[[IDX:.*]]: index
// CHECK: %[[PASS_THROUGH:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[MASK:.*]] = vector.create_mask %[[DIM_2]], %[[DIM_1]] : vector<2x4xi1>
// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[IDX]], %[[IDX]]], %[[PASS_THROUGH]], %[[MASK]] {in_bounds = [true, true]} : memref<?x?xf32>, vector<2x4xf32>
// CHECK: %[[BCAST:.*]] = vector.broadcast %[[T_READ]] : vector<2x4xf32> to vector<8x2x4xf32>
// CHECK: %[[TRANSPOSE:.*]] = vector.transpose %[[BCAST]], [0, 2, 1] : vector<8x2x4xf32> to vector<8x4x2xf32>
// CHECK: return %[[TRANSPOSE]] : vector<8x4x2xf32>
-func.func @xfer_read_minor_identity_tranposed_with_mask(
+func.func @xfer_read_minor_identity_transposed_with_mask(
%mem: memref<?x?xf32>,
- %dim_1: index,
- %dim_2: index,
+ %mask: vector<2x4xi1>,
%idx: index) -> (vector<8x4x2xf32>) {
%pad = arith.constant 0.000000e+00 : f32
- %mask = vector.create_mask %dim_2, %dim_1 : vector<2x4xi1>
%res = vector.transfer_read %mem[%idx, %idx], %pad, %mask {
in_bounds = [true, true, true],
permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>
@@ -290,24 +291,22 @@ func.func @xfer_read_minor_identity_tranposed_with_mask(
return %res : vector<8x4x2xf32>
}
-// CHECK-LABEL: func.func @xfer_read_minor_identity_tranposed_with_mask_scalable(
+// CHECK-LABEL: func.func @xfer_read_minor_identity_transposed_with_mask_scalable(
// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,
-// CHECK-SAME: %[[DIM_1:.*]]: index, %[[DIM_2:.*]]: index, %[[IDX:.*]]: index) -> vector<8x[4]x2xf32> {
+// CHECK-SAME: %[[MASK:.*]]: vector<2x[4]xi1>
+// CHECK-SAME: %[[IDX:.*]]: index
// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[MASK:.*]] = vector.create_mask %[[DIM_2]], %[[DIM_1]] : vector<2x[4]xi1>
// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[IDX]], %[[IDX]]], %[[PAD]], %[[MASK]] {in_bounds = [true, true]} : memref<?x?xf32>, vector<2x[4]xf32>
// CHECK: %[[BCAST:.*]] = vector.broadcast %[[T_READ]] : vector<2x[4]xf32> to vector<8x2x[4]xf32>
// CHECK: %[[TRANSPOSE:.*]] = vector.transpose %[[BCAST]], [0, 2, 1] : vector<8x2x[4]xf32> to vector<8x[4]x2xf32>
// CHECK: return %[[TRANSPOSE]] : vector<8x[4]x2xf32>
-func.func @xfer_read_minor_identity_tranposed_with_mask_scalable(
+func.func @xfer_read_minor_identity_transposed_with_mask_scalable(
%mem: memref<?x?xf32>,
- %dim_1: index,
- %dim_2: index,
+ %mask: vector<2x[4]xi1>,
%idx: index) -> (vector<8x[4]x2xf32>) {
%pad = arith.constant 0.000000e+00 : f32
- %mask = vector.create_mask %dim_2, %dim_1 : vector<2x[4]xi1>
%res = vector.transfer_read %mem[%idx, %idx], %pad, %mask {
in_bounds = [true, true, true],
permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>
@@ -319,24 +318,26 @@ func.func @xfer_read_minor_identity_tranposed_with_mask_scalable(
// Masked version is not supported
// CHECK-LABEL: func @xfer_read_minor_identity_transposed_masked(
-// CHECK-SAME: %[[DEST:.*]]: tensor<?x1xf32>,
-// CHECK-SAME: %[[MASK:.*]]: vector<4x1xi1>
+// CHECK-SAME: %[[DEST:.*]]: tensor<?x?xf32>,
+// CHECK-SAME: %[[MASK:.*]]: vector<2x4xi1>
+// CHECK-SAME: %[[IDX:.*]]: index
// CHECK-NOT: vector.transpose
-// CHECK: vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{.*}}: tensor<?x1xf32>, vector<1x4x4xf32> } : vector<4x1xi1> -> vector<1x4x4xf32>
+// CHECK: vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{.*}}: tensor<?x?xf32>, vector<8x4x2xf32> } : vector<2x4xi1> -> vector<8x4x2xf32>
func.func @xfer_read_minor_identity_transposed_masked(
- %dest: tensor<?x1xf32>,
- %mask : vector<4x1xi1>,
- %idx: index) {
+ %dest: tensor<?x?xf32>,
+ %mask: vector<2x4xi1>,
+ %idx: index) -> (vector<8x4x2xf32>) {
%pad = arith.constant 0.000000e+00 : f32
- %3 = vector.mask %mask {
+
+ %res = vector.mask %mask {
vector.transfer_read %dest[%idx, %idx], %pad {
- permutation_map = affine_map<(d0, d1) -> (d1, 0, d0)>
- } : tensor<?x1xf32>, vector<1x4x4xf32>
- } : vector<4x1xi1> -> vector<1x4x4xf32>
+ in_bounds = [true, true, true],
+ permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>
+ } : tensor<?x?xf32>, vector<8x4x2xf32>
+ } : vector<2x4xi1> -> vector<8x4x2xf32>
- "test.some_use"(%3) : (vector<1x4x4xf32>) -> ()
- return
+ return %res : vector<8x4x2xf32>
}
// CHECK-LABEL: func.func @xfer_read_minor_identity_transposed_masked_scalable(
@@ -346,7 +347,7 @@ func.func @xfer_read_minor_identity_transposed_masked(
// CHECK: %[[T_READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{.*}} : tensor<?x?xf32>, vector<8x[4]x2xf32> } : vector<2x[4]xi1> -> vector<8x[4]x2xf32>
func.func @xfer_read_minor_identity_transposed_masked_scalable(
%dest: tensor<?x?xf32>,
- %mask : vector<2x[4]xi1>,
+ %mask: vector<2x[4]xi1>,
%idx: index) -> vector<8x[4]x2xf32> {
%pad = arith.constant 0.000000e+00 : f32
@@ -388,17 +389,16 @@ func.func @xfer_read_minor_identitiy_bcast_dims_scalable(
// CHECK-LABEL: func.func @xfer_read_minor_identitiy_bcast_dims_masked
// CHECK-SAME: %[[MEM:.*]]: memref<?x?x?x?xf32>,
-// CHECK-SAME: %[[DIM:.*]]: index,
+// CHECK-SAME: %[[MASK:.*]]: vector<[4]x3xi1>
// CHECK-SAME: %[[IDX:.*]]: index) -> vector<8x[4]x2x3xf32> {
// CHECK-NOT: vector.broadcast
-// CHECK: %[[MASK:.*]] = vector.mask %0 { vector.transfer_read %[[MEM]]{{.*}} : memref<?x?x?x?xf32>, vector<8x[4]x2x3xf32> } : vector<[4]x3xi1> -> vector<8x[4]x2x3xf32>
+// CHECK: vector.mask %[[MASK]] { vector.transfer_read %[[MEM]]{{.*}} : memref<?x?x?x?xf32>, vector<8x[4]x2x3xf32> } : vector<[4]x3xi1> -> vector<8x[4]x2x3xf32>
func.func @xfer_read_minor_identitiy_bcast_dims_masked(
%mem: memref<?x?x?x?xf32>,
- %dim: index,
+ %mask: vector<[4]x3xi1>,
%idx: index) -> vector<8x[4]x2x3xf32> {
%pad = arith.constant 0.000000e+00 : f32
- %mask = vector.create_mask %dim, %dim: vector<[4]x3xi1>
%res = vector.mask %mask {
vector.transfer_read %mem[%idx, %idx, %idx, %idx], %pad {
>From 86553788780ec3992eb0ab18815cc12f2782be1a Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy at intel.com>
Date: Tue, 18 Feb 2025 12:03:37 +0100
Subject: [PATCH 09/31] [SPIR-V] Built-in variables: mapping from an OpenCL to
SPIR-V BuiltIn and SPIR-V friendly builtins for Image Read/Write instructions
(#127242)
This PR improves built-in variables and functions support:
* extends mapping from an OpenCL C built-in function to the SPIR-V
BuiltIn variables as in
https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_Env.html#_built_in_variables,
and
* adds SPIR-V friendly builtins for Image Read/Write instructions.
Test cases are extended accordingly.
---
llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 16 +++-
llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 12 +++
.../CodeGen/SPIRV/builtin_vars-decorate.ll | 85 ++++++++++++++-----
.../CodeGen/SPIRV/transcoding/builtin_vars.ll | 9 +-
.../CodeGen/SPIRV/transcoding/spirv-types.ll | 58 ++++++++++++-
5 files changed, 153 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index aec2aaccb33a8..7b897f7e34c6f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -216,6 +216,7 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall,
// Check if the extracted name begins with:
// - "__spirv_ImageSampleExplicitLod"
// - "__spirv_ImageRead"
+ // - "__spirv_ImageWrite"
// - "__spirv_ImageQuerySizeLod"
// - "__spirv_UDotKHR"
// - "__spirv_SDotKHR"
@@ -233,20 +234,21 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall,
// - "__spirv_SConvert"
// - "__spirv_FConvert"
// - "__spirv_SatConvert"
- // and contains return type information at the end "_R<type>".
+ // and maybe contains return type information at the end "_R<type>".
// If so, extract the plain builtin name without the type information.
static const std::regex SpvWithR(
- "(__spirv_(ImageSampleExplicitLod|ImageRead|ImageQuerySizeLod|UDotKHR|"
+ "(__spirv_(ImageSampleExplicitLod|ImageRead|ImageWrite|ImageQuerySizeLod|"
+ "UDotKHR|"
"SDotKHR|SUDotKHR|SDotAccSatKHR|UDotAccSatKHR|SUDotAccSatKHR|"
"ReadClockKHR|SubgroupBlockReadINTEL|SubgroupImageBlockReadINTEL|"
"SubgroupImageMediaBlockReadINTEL|SubgroupImageMediaBlockWriteINTEL|"
"Convert|"
- "UConvert|SConvert|FConvert|SatConvert).*)_R[^_]*_?(\\w+)?.*");
+ "UConvert|SConvert|FConvert|SatConvert)[^_]*)(_R[^_]*_?(\\w+)?.*)?");
std::smatch Match;
if (std::regex_match(BuiltinName, Match, SpvWithR) && Match.size() > 1) {
std::ssub_match SubMatch;
if (DecorationId && Match.size() > 3) {
- SubMatch = Match[3];
+ SubMatch = Match[4];
*DecorationId = demangledPostfixToDecorationId(SubMatch.str());
}
SubMatch = Match[1];
@@ -1932,6 +1934,9 @@ static bool generateReadImageInst(const StringRef DemangledCall,
const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpImageRead, Call,
+ GR->getSPIRVTypeID(Call->ReturnType));
Register Image = Call->Arguments[0];
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
bool HasOclSampler = DemangledCall.contains_insensitive("ocl_sampler");
@@ -2011,6 +2016,9 @@ static bool generateReadImageInst(const StringRef DemangledCall,
static bool generateWriteImageInst(const SPIRV::IncomingCall *Call,
MachineIRBuilder &MIRBuilder,
SPIRVGlobalRegistry *GR) {
+ if (Call->isSpirvOp())
+ return buildOpFromWrapper(MIRBuilder, SPIRV::OpImageWrite, Call,
+ Register(0));
MIRBuilder.buildInstr(SPIRV::OpImageWrite)
.addUse(Call->Arguments[0]) // Image.
.addUse(Call->Arguments[1]) // Coordinate.
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 16f4252173e33..85f42fc08a4e0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -110,11 +110,13 @@ def : DemangledBuiltin<"__spirv_DotAccSat", OpenCL_std, IntegerDot, 3, 3>;
def : DemangledBuiltin<"read_imagei", OpenCL_std, ReadImage, 2, 4>;
def : DemangledBuiltin<"read_imageui", OpenCL_std, ReadImage, 2, 4>;
def : DemangledBuiltin<"read_imagef", OpenCL_std, ReadImage, 2, 4>;
+def : DemangledBuiltin<"__spirv_ImageRead", OpenCL_std, ReadImage, 2, 0>;
def : DemangledBuiltin<"write_imagef", OpenCL_std, WriteImage, 3, 4>;
def : DemangledBuiltin<"write_imagei", OpenCL_std, WriteImage, 3, 4>;
def : DemangledBuiltin<"write_imageui", OpenCL_std, WriteImage, 3, 4>;
def : DemangledBuiltin<"write_imageh", OpenCL_std, WriteImage, 3, 4>;
+def : DemangledBuiltin<"__spirv_ImageWrite", OpenCL_std, WriteImage, 3, 0>;
def : DemangledBuiltin<"__translate_sampler_initializer", OpenCL_std, SampleImage, 1, 1>;
def : DemangledBuiltin<"__spirv_SampledImage", OpenCL_std, SampleImage, 2, 2>;
@@ -1323,6 +1325,15 @@ multiclass DemangledGetBuiltin<string name, InstructionSet set, BuiltinGroup gro
}
// Builtin variable records:
+defm : DemangledGetBuiltin<"get_global_linear_id", OpenCL_std, Variable, GlobalLinearId>;
+defm : DemangledGetBuiltin<"get_local_linear_id", OpenCL_std, Variable, LocalInvocationIndex>;
+defm : DemangledGetBuiltin<"get_work_dim", OpenCL_std, Variable, WorkDim>;
+defm : DemangledGetBuiltin<"get_sub_group_size", OpenCL_std, Variable, SubgroupSize>;
+defm : DemangledGetBuiltin<"get_max_sub_group_size", OpenCL_std, Variable, SubgroupMaxSize>;
+defm : DemangledGetBuiltin<"get_num_sub_groups", OpenCL_std, Variable, NumSubgroups>;
+defm : DemangledGetBuiltin<"get_enqueued_num_sub_groups", OpenCL_std, Variable, NumEnqueuedSubgroups>;
+defm : DemangledGetBuiltin<"get_sub_group_id", OpenCL_std, Variable, SubgroupId>;
+defm : DemangledGetBuiltin<"get_sub_group_local_id", OpenCL_std, Variable, SubgroupLocalInvocationId>;
defm : DemangledGetBuiltin<"get_sub_group_eq_mask", OpenCL_std, Variable, SubgroupEqMask>;
defm : DemangledGetBuiltin<"get_sub_group_ge_mask", OpenCL_std, Variable, SubgroupGeMask>;
defm : DemangledGetBuiltin<"get_sub_group_gt_mask", OpenCL_std, Variable, SubgroupGtMask>;
@@ -1339,6 +1350,7 @@ defm : DemangledGetBuiltin<"get_global_size", OpenCL_std, GetQuery, GlobalSize>;
defm : DemangledGetBuiltin<"get_group_id", OpenCL_std, GetQuery, WorkgroupId>;
defm : DemangledGetBuiltin<"get_enqueued_local_size", OpenCL_std, GetQuery, EnqueuedWorkgroupSize>;
defm : DemangledGetBuiltin<"get_num_groups", OpenCL_std, GetQuery, NumWorkgroups>;
+defm : DemangledGetBuiltin<"get_global_offset", OpenCL_std, GetQuery, GlobalOffset>;
defm : DemangledGetBuiltin<"__hlsl_wave_get_lane_index", GLSL_std_450, Wave, SubgroupLocalInvocationId>;
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/SPIRV/builtin_vars-decorate.ll b/llvm/test/CodeGen/SPIRV/builtin_vars-decorate.ll
index 59abd5dbee6a0..0c9b29de890d4 100644
--- a/llvm/test/CodeGen/SPIRV/builtin_vars-decorate.ll
+++ b/llvm/test/CodeGen/SPIRV/builtin_vars-decorate.ll
@@ -1,22 +1,23 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
-
-; CHECK: OpName %[[#WD:]] "__spirv_BuiltInWorkDim"
-; CHECK: OpName %[[#GS:]] "__spirv_BuiltInGlobalSize"
-; CHECK: OpName %[[#GII:]] "__spirv_BuiltInGlobalInvocationId"
-; CHECK: OpName %[[#WS:]] "__spirv_BuiltInWorkgroupSize"
-; CHECK: OpName %[[#EWS:]] "__spirv_BuiltInEnqueuedWorkgroupSize"
-; CHECK: OpName %[[#LLI:]] "__spirv_BuiltInLocalInvocationId"
-; CHECK: OpName %[[#NW:]] "__spirv_BuiltInNumWorkgroups"
-; CHECK: OpName %[[#WI:]] "__spirv_BuiltInWorkgroupId"
-; CHECK: OpName %[[#GO:]] "__spirv_BuiltInGlobalOffset"
-; CHECK: OpName %[[#GLI:]] "__spirv_BuiltInGlobalLinearId"
-; CHECK: OpName %[[#LLII:]] "__spirv_BuiltInLocalInvocationIndex"
-; CHECK: OpName %[[#SS:]] "__spirv_BuiltInSubgroupSize"
-; CHECK: OpName %[[#SMS:]] "__spirv_BuiltInSubgroupMaxSize"
-; CHECK: OpName %[[#NS:]] "__spirv_BuiltInNumSubgroups"
-; CHECK: OpName %[[#NES:]] "__spirv_BuiltInNumEnqueuedSubgroups"
-; CHECK: OpName %[[#SI:]] "__spirv_BuiltInSubgroupId"
-; CHECK: OpName %[[#SLII:]] "__spirv_BuiltInSubgroupLocalInvocationId"
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[#WD:]] "__spirv_BuiltInWorkDim"
+; CHECK-DAG: OpName %[[#GS:]] "__spirv_BuiltInGlobalSize"
+; CHECK-DAG: OpName %[[#GII:]] "__spirv_BuiltInGlobalInvocationId"
+; CHECK-DAG: OpName %[[#WS:]] "__spirv_BuiltInWorkgroupSize"
+; CHECK-DAG: OpName %[[#EWS:]] "__spirv_BuiltInEnqueuedWorkgroupSize"
+; CHECK-DAG: OpName %[[#LLI:]] "__spirv_BuiltInLocalInvocationId"
+; CHECK-DAG: OpName %[[#NW:]] "__spirv_BuiltInNumWorkgroups"
+; CHECK-DAG: OpName %[[#WI:]] "__spirv_BuiltInWorkgroupId"
+; CHECK-DAG: OpName %[[#GO:]] "__spirv_BuiltInGlobalOffset"
+; CHECK-DAG: OpName %[[#GLI:]] "__spirv_BuiltInGlobalLinearId"
+; CHECK-DAG: OpName %[[#LLII:]] "__spirv_BuiltInLocalInvocationIndex"
+; CHECK-DAG: OpName %[[#SS:]] "__spirv_BuiltInSubgroupSize"
+; CHECK-DAG: OpName %[[#SMS:]] "__spirv_BuiltInSubgroupMaxSize"
+; CHECK-DAG: OpName %[[#NS:]] "__spirv_BuiltInNumSubgroups"
+; CHECK-DAG: OpName %[[#NES:]] "__spirv_BuiltInNumEnqueuedSubgroups"
+; CHECK-DAG: OpName %[[#SI:]] "__spirv_BuiltInSubgroupId"
+; CHECK-DAG: OpName %[[#SLII:]] "__spirv_BuiltInSubgroupLocalInvocationId"
; CHECK-DAG: OpDecorate %[[#NW]] BuiltIn NumWorkgroups
; CHECK-DAG: OpDecorate %[[#WS]] BuiltIn WorkgroupSize
@@ -35,6 +36,33 @@
; CHECK-DAG: OpDecorate %[[#NES]] BuiltIn NumEnqueuedSubgroups
; CHECK-DAG: OpDecorate %[[#SI]] BuiltIn SubgroupId
; CHECK-DAG: OpDecorate %[[#SLII]] BuiltIn SubgroupLocalInvocationId
+
+; CHECK-DAG: %[[#SizeT:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#Int32:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#SizeTPtr:]] = OpTypePointer Input %[[#SizeT]]
+; CHECK-DAG: %[[#Int32Ptr:]] = OpTypePointer Input %[[#Int32]]
+
+; CHECK-DAG: %[[#GLI]] = OpVariable %[[#SizeTPtr]] Input
+; CHECK-DAG: %[[#LLII]] = OpVariable %[[#SizeTPtr]] Input
+; CHECK-DAG: %[[#WD]] = OpVariable %[[#Int32Ptr]] Input
+; CHECK-DAG: %[[#SS]] = OpVariable %[[#Int32Ptr]] Input
+; CHECK-DAG: %[[#SMS]] = OpVariable %[[#Int32Ptr]] Input
+; CHECK-DAG: %[[#NS]] = OpVariable %[[#Int32Ptr]] Input
+; CHECK-DAG: %[[#NES]] = OpVariable %[[#Int32Ptr]] Input
+; CHECK-DAG: %[[#SI]] = OpVariable %[[#Int32Ptr]] Input
+; CHECK-DAG: %[[#SLII]] = OpVariable %[[#Int32Ptr]] Input
+
+; CHECK: OpFunction
+; CHECK: %[[#]] = OpLoad %[[#SizeT]] %[[#GLI]]
+; CHECK: %[[#]] = OpLoad %[[#SizeT]] %[[#LLII]]
+; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#WD]]
+; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#SS]]
+; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#SMS]]
+; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#NS]]
+; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#NES]]
+; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#SI]]
+; CHECK: %[[#]] = OpLoad %[[#Int32]] %[[#SLII]]
+
@__spirv_BuiltInWorkDim = external addrspace(1) global i32
@__spirv_BuiltInGlobalSize = external addrspace(1) global <3 x i32>
@__spirv_BuiltInGlobalInvocationId = external addrspace(1) global <3 x i32>
@@ -55,5 +83,24 @@
define spir_kernel void @_Z1wv() {
entry:
+ %r1 = tail call spir_func i64 @get_global_linear_id()
+ %r2 = tail call spir_func i64 @get_local_linear_id()
+ %r3 = tail call spir_func i32 @get_work_dim()
+ %r4 = tail call spir_func i32 @get_sub_group_size()
+ %r5 = tail call spir_func i32 @get_max_sub_group_size()
+ %r6 = tail call spir_func i32 @get_num_sub_groups()
+ %r7 = tail call spir_func i32 @get_enqueued_num_sub_groups()
+ %r8 = tail call spir_func i32 @get_sub_group_id()
+ %r9 = tail call spir_func i32 @get_sub_group_local_id()
ret void
}
+
+declare spir_func i64 @get_global_linear_id()
+declare spir_func i64 @get_local_linear_id()
+declare spir_func i32 @get_work_dim()
+declare spir_func i32 @get_sub_group_size()
+declare spir_func i32 @get_max_sub_group_size()
+declare spir_func i32 @get_num_sub_groups()
+declare spir_func i32 @get_enqueued_num_sub_groups()
+declare spir_func i32 @get_sub_group_id()
+declare spir_func i32 @get_sub_group_local_id()
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll
index 5074893163565..44d2f5e24f59d 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/builtin_vars.ll
@@ -1,8 +1,11 @@
-; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
-; CHECK-SPIRV: OpDecorate %[[#Id:]] BuiltIn GlobalLinearId
-; CHECK-SPIRV: %[[#Id:]] = OpVariable %[[#]]
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK: OpDecorate %[[#Id:]] BuiltIn GlobalLinearId
+; CHECK: %[[#Id:]] = OpVariable %[[#]]
@__spirv_BuiltInGlobalLinearId = external addrspace(1) global i32
diff --git a/llvm/test/CodeGen/SPIRV/transcoding/spirv-types.ll b/llvm/test/CodeGen/SPIRV/transcoding/spirv-types.ll
index 8d99a0c6cd1ce..36ae6bf478127 100644
--- a/llvm/test/CodeGen/SPIRV/transcoding/spirv-types.ll
+++ b/llvm/test/CodeGen/SPIRV/transcoding/spirv-types.ll
@@ -85,5 +85,61 @@ define spir_func void @test_sampler(target("spirv.Image", float, 1, 1, 0, 0, 0,
}
declare spir_func target("spirv.Image", float, 1, 1, 0, 0, 0, 0, 0) @_Z20__spirv_SampledImagePU3AS1K34__spirv_Image__float_1_1_0_0_0_0_0PU3AS1K15__spirv_Sampler(target("spirv.Image", float, 1, 1, 0, 0, 0, 0, 0), target("spirv.Sampler"))
-
declare spir_func <4 x float> @_Z38__spirv_ImageSampleExplicitLod_Rfloat4PU3AS120__spirv_SampledImageDv4_iif(target("spirv.Image", float, 1, 1, 0, 0, 0, 0, 0), <4 x i32>, i32, float)
+
+; CHECK-SPIRV: %[[#]] = OpImageRead
+; CHECK-SPIRV: %[[#]] = OpImageRead
+; CHECK-SPIRV: %[[#]] = OpImageRead
+; CHECK-SPIRV: %[[#]] = OpImageRead
+; CHECK-SPIRV: %[[#]] = OpImageRead
+; CHECK-SPIRV: %[[#]] = OpImageRead
+; CHECK-SPIRV: %[[#]] = OpImageRead
+; CHECK-SPIRV: %[[#]] = OpImageSampleExplicitLod
+
+define dso_local spir_kernel void @reads() {
+ %1 = tail call spir_func i32 @_Z17__spirv_ImageReadIi14ocl_image3d_roDv4_iET_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 0) poison, <4 x i32> zeroinitializer)
+ %2 = tail call spir_func <2 x i32> @_Z17__spirv_ImageReadIDv2_i14ocl_image2d_roS0_ET_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison, <2 x i32> zeroinitializer)
+ %3 = tail call spir_func <4 x i32> @_Z17__spirv_ImageReadIDv4_j14ocl_image3d_roDv4_iET_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 0) poison, <4 x i32> zeroinitializer)
+ %4 = tail call spir_func signext i16 @_Z17__spirv_ImageReadIs14ocl_image1d_roiET_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 0) poison, i32 0)
+ %5 = tail call spir_func zeroext i16 @_Z17__spirv_ImageReadIt14ocl_image3d_roDv4_iET_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 0) poison, <4 x i32> zeroinitializer)
+ %6 = tail call spir_func <2 x float> @_Z17__spirv_ImageReadIDv2_f14ocl_image1d_roiET_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 0) poison, i32 0)
+ %7 = tail call spir_func half @_Z17__spirv_ImageReadIDF16_14ocl_image2d_roDv2_iET_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0) poison, <2 x i32> zeroinitializer)
+ %8 = tail call spir_func <4 x i32> @_Z30__spirv_ImageSampleExplicitLodI32__spirv_SampledImage__image1d_roDv4_jfET0_T_T1_if(target("spirv.SampledImage", void, 0, 0, 0, 0, 0, 0, 0) poison, float 0.000000e+00, i32 2, float 0.000000e+00)
+ ret void
+}
+
+declare dso_local spir_func i32 @_Z17__spirv_ImageReadIi14ocl_image3d_roDv4_iET_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 0), <4 x i32>)
+declare dso_local spir_func <2 x i32> @_Z17__spirv_ImageReadIDv2_i14ocl_image2d_roS0_ET_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0), <2 x i32>)
+declare dso_local spir_func <4 x i32> @_Z17__spirv_ImageReadIDv4_j14ocl_image3d_roDv4_iET_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 0), <4 x i32>)
+declare dso_local spir_func signext i16 @_Z17__spirv_ImageReadIs14ocl_image1d_roiET_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 0), i32)
+declare dso_local spir_func zeroext i16 @_Z17__spirv_ImageReadIt14ocl_image3d_roDv4_iET_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 0), <4 x i32>)
+declare dso_local spir_func <2 x float> @_Z17__spirv_ImageReadIDv2_f14ocl_image1d_roiET_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 0), i32)
+declare dso_local spir_func half @_Z17__spirv_ImageReadIDF16_14ocl_image2d_roDv2_iET_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 0), <2 x i32>)
+declare dso_local spir_func <4 x i32> @_Z30__spirv_ImageSampleExplicitLodI32__spirv_SampledImage__image1d_roDv4_jfET0_T_T1_if(target("spirv.SampledImage", void, 0, 0, 0, 0, 0, 0, 0), float noundef, i32 noundef, float noundef)
+
+; CHECK-SPIRV: OpImageWrite
+; CHECK-SPIRV: OpImageWrite
+; CHECK-SPIRV: OpImageWrite
+; CHECK-SPIRV: OpImageWrite
+; CHECK-SPIRV: OpImageWrite
+; CHECK-SPIRV: OpImageWrite
+; CHECK-SPIRV: OpImageWrite
+
+define dso_local spir_kernel void @writes() {
+ call spir_func void @_Z18__spirv_ImageWriteI14ocl_image3d_woDv4_iiEvT_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 1) poison, <4 x i32> zeroinitializer, i32 zeroinitializer)
+ call spir_func void @_Z18__spirv_ImageWriteI14ocl_image2d_woDv2_iS1_EvT_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) poison, <2 x i32> zeroinitializer, <2 x i32> zeroinitializer)
+ call spir_func void @_Z18__spirv_ImageWriteI14ocl_image3d_woDv4_iDv4_jEvT_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 1) poison, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer)
+ call spir_func void @_Z18__spirv_ImageWriteI14ocl_image1d_woisEvT_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 1) poison, i32 0, i16 signext 0)
+ call spir_func void @_Z18__spirv_ImageWriteI14ocl_image3d_woDv4_itEvT_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 1) poison, <4 x i32> zeroinitializer, i16 zeroext 0)
+ call spir_func void @_Z18__spirv_ImageWriteI14ocl_image1d_woiDv2_fEvT_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 1) poison, i32 0, <2 x float> zeroinitializer)
+ call spir_func void @_Z18__spirv_ImageWriteI14ocl_image2d_woDv2_iDF16_EvT_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1) poison, <2 x i32> zeroinitializer, half zeroinitializer)
+ ret void
+}
+
+declare dso_local spir_func void @_Z18__spirv_ImageWriteI14ocl_image3d_woDv4_iiEvT_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 1), <4 x i32>, i32)
+declare dso_local spir_func void @_Z18__spirv_ImageWriteI14ocl_image2d_woDv2_iS1_EvT_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, <2 x i32>)
+declare dso_local spir_func void @_Z18__spirv_ImageWriteI14ocl_image3d_woDv4_iDv4_jEvT_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 1), <4 x i32>, <4 x i32>)
+declare dso_local spir_func void @_Z18__spirv_ImageWriteI14ocl_image1d_woisEvT_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 1), i32, i16 signext)
+declare dso_local spir_func void @_Z18__spirv_ImageWriteI14ocl_image3d_woDv4_itEvT_T0_T1_(target("spirv.Image", void, 2, 0, 0, 0, 0, 0, 1), <4 x i32>, i16 zeroext)
+declare dso_local spir_func void @_Z18__spirv_ImageWriteI14ocl_image1d_woiDv2_fEvT_T0_T1_(target("spirv.Image", void, 0, 0, 0, 0, 0, 0, 1), i32, <2 x float>)
+declare dso_local spir_func void @_Z18__spirv_ImageWriteI14ocl_image2d_woDv2_iDF16_EvT_T0_T1_(target("spirv.Image", void, 1, 0, 0, 0, 0, 0, 1), <2 x i32>, half)
>From 762001118c068317ec67274221497be2e6499c6a Mon Sep 17 00:00:00 2001
From: Steven Cooreman <steven.cooreman at gmail.com>
Date: Tue, 18 Feb 2025 12:12:23 +0100
Subject: [PATCH 10/31] [libc++] Do not guard inclusion of wchar.h with
_LIBCPP_HAS_WIDE_CHARACTERS (#126924)
`mbstate_t` needs to be visible to libcpp, even when it is not providing
wide
character functionality (i.e. `_LIBCPP_HAS_WIDE_CHARACTERS` is turned
off)
and thus not using any of the C library's wide character functions.
There are C libraries (such as newlib-nano/nanolib/picolibc) which do
provide their definition of `mbstate_t` in `<wchar.h>` even though they
do not
come with wide character functions.
Since there is a way to conditionally include the C library's
`<wchar.h>`
only if it exists, we should rely on the fact that if it exists, it will
provide `mbstate_t`. Removing this guard will allow using libc++ on top
of
newlib-nano/picolibc while not breaking the cases where it is used on
top
of a C library which doesn't provide `<wchar.h>` (since it would then
still
go look for `<uchar.h>` or error out).
---
libcxx/include/__mbstate_t.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/libcxx/include/__mbstate_t.h b/libcxx/include/__mbstate_t.h
index e013384454b41..c23ea7113ca70 100644
--- a/libcxx/include/__mbstate_t.h
+++ b/libcxx/include/__mbstate_t.h
@@ -43,12 +43,12 @@
# include <bits/types/mbstate_t.h> // works on most Unixes
#elif __has_include(<sys/_types/_mbstate_t.h>)
# include <sys/_types/_mbstate_t.h> // works on Darwin
-#elif _LIBCPP_HAS_WIDE_CHARACTERS && __has_include_next(<wchar.h>)
-# include_next <wchar.h> // fall back to the C standard provider of mbstate_t
+#elif __has_include_next(<wchar.h>)
+# include_next <wchar.h> // use the C standard provider of mbstate_t if present
#elif __has_include_next(<uchar.h>)
-# include_next <uchar.h> // <uchar.h> is also required to make mbstate_t visible
+# include_next <uchar.h> // Try <uchar.h> in absence of <wchar.h> for mbstate_t
#else
-# error "We don't know how to get the definition of mbstate_t without <wchar.h> on your platform."
+# error "We don't know how to get the definition of mbstate_t on your platform."
#endif
#endif // _LIBCPP___MBSTATE_T_H
>From 059f044309a282447fb25073875cff8d2bd96f78 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames at gmail.com>
Date: Tue, 18 Feb 2025 22:10:17 +1100
Subject: [PATCH 11/31] [ORC] Propagate weak & hidden flags when creating lazy
reexports, redirectables.
Updates JITLinkRedirectableSymbolManager to take alias flags into account when
setting the scope and linkage of the created stubs (weak aliases get now get weak
linkage, hidden stubs get hidden visibility).
Updates lazyReexports to propagate alias flags (rather than trampoline flags)
when building the initial destinations map for the redirectable symbols manager.
Together these changes allow the LazyObjectLinkingLayer to link objects
containing weak and hidden symbols.
---
.../orc/TestCases/Generic/Inputs/bar-ret-void-weak.ll | 4 ++++
.../TestCases/Generic/Inputs/baz-ret-void-hidden.ll | 4 ++++
compiler-rt/test/orc/TestCases/Generic/lazy-link.ll | 10 +++++++++-
.../Orc/JITLinkRedirectableSymbolManager.cpp | 5 ++++-
llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp | 2 +-
5 files changed, 22 insertions(+), 3 deletions(-)
create mode 100644 compiler-rt/test/orc/TestCases/Generic/Inputs/bar-ret-void-weak.ll
create mode 100644 compiler-rt/test/orc/TestCases/Generic/Inputs/baz-ret-void-hidden.ll
diff --git a/compiler-rt/test/orc/TestCases/Generic/Inputs/bar-ret-void-weak.ll b/compiler-rt/test/orc/TestCases/Generic/Inputs/bar-ret-void-weak.ll
new file mode 100644
index 0000000000000..7301b43e7c92d
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Generic/Inputs/bar-ret-void-weak.ll
@@ -0,0 +1,4 @@
+define weak void @bar() {
+entry:
+ ret void
+}
diff --git a/compiler-rt/test/orc/TestCases/Generic/Inputs/baz-ret-void-hidden.ll b/compiler-rt/test/orc/TestCases/Generic/Inputs/baz-ret-void-hidden.ll
new file mode 100644
index 0000000000000..27e19deea6ebd
--- /dev/null
+++ b/compiler-rt/test/orc/TestCases/Generic/Inputs/baz-ret-void-hidden.ll
@@ -0,0 +1,4 @@
+define hidden void @baz() {
+entry:
+ ret void
+}
diff --git a/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll b/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
index 5a8dbfc532b0f..1c375bcf1e62f 100644
--- a/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
+++ b/compiler-rt/test/orc/TestCases/Generic/lazy-link.ll
@@ -6,9 +6,11 @@
; RUN: rm -rf %t && mkdir -p %t
; RUN: %clang -c -o %t/foo.o %S/Inputs/foo-ret-42.ll
; RUN: %clang -c -o %t/x.o %S/Inputs/var-x-42.ll
+; RUN: %clang -c -o %t/bar.o %S/Inputs/bar-ret-void-weak.ll
+; RUN: %clang -c -o %t/baz.o %S/Inputs/baz-ret-void-hidden.ll
; RUN: %clang -c -o %t/main.o %s
; RUN: %llvm_jitlink -noexec -show-linked-files %t/main.o -lazy %t/foo.o \
-; RUN: -lazy %t/x.o | FileCheck %s
+; RUN: -lazy %t/x.o -lazy %t/bar.o -lazy %t/baz.o | FileCheck %s
;
; UNSUPPORTED: system-windows
; REQUIRES: target={{(arm|aarch|x86_)64.*}}
@@ -21,9 +23,15 @@
declare i32 @foo()
@x = external global i32
+declare void @bar()
+declare hidden void @baz()
+
+
define i32 @main(i32 %argc, ptr %argv) {
entry:
%foo_result = call i32 @foo()
+ call void @bar()
+ call void @baz()
%x_val = load i32, ptr @x
%result = add nsw i32 %foo_result, %x_val
ret i32 %result
diff --git a/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
index 06c545d62d76a..b5b380971d204 100644
--- a/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.cpp
@@ -47,7 +47,10 @@ void JITLinkRedirectableSymbolManager::emitRedirectableSymbols(
Ptr.setScope(jitlink::Scope::Hidden);
auto &Stub = PtrJumpStubCreator(*G, StubsSection, Ptr);
Stub.setName(Name);
- Stub.setScope(jitlink::Scope::Default);
+ Stub.setScope(Def.getFlags().isExported() ? jitlink::Scope::Default
+ : jitlink::Scope::Hidden);
+ Stub.setLinkage(!Def.getFlags().isWeak() ? jitlink::Linkage::Strong
+ : jitlink::Linkage::Weak);
NewSymbols[std::move(PtrName)] = JITSymbolFlags();
}
diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
index 7b38621eba824..80f2a1304dde7 100644
--- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -387,7 +387,7 @@ void LazyReexportsManager::emitRedirectableSymbols(
SymbolMap Redirs;
size_t I = 0;
for (auto &[Name, AI] : Reexports)
- Redirs[Name] = (*ReentryPoints)[I++];
+ Redirs[Name] = {(*ReentryPoints)[I++].getAddress(), AI.AliasFlags};
I = 0;
if (!Reexports.empty()) {
>From 719c46b35958782622e300696fbe6af6549b4cdc Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me at antoniofrighetto.com>
Date: Tue, 18 Feb 2025 12:42:37 +0100
Subject: [PATCH 12/31] [FunctionAttrs] Fix typo in `getArgumentAccessInfo`
name (NFC)
---
llvm/lib/Transforms/IPO/FunctionAttrs.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index a66d7ce9c3f50..02b0fcb3981a7 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -651,9 +651,9 @@ struct ArgumentUsesSummary {
SmallDenseMap<const BasicBlock *, UsesPerBlockInfo, 16> UsesPerBlock;
};
-ArgumentAccessInfo getArgmentAccessInfo(const Instruction *I,
- const ArgumentUse &ArgUse,
- const DataLayout &DL) {
+ArgumentAccessInfo getArgumentAccessInfo(const Instruction *I,
+ const ArgumentUse &ArgUse,
+ const DataLayout &DL) {
auto GetTypeAccessRange =
[&DL](Type *Ty,
std::optional<int64_t> Offset) -> std::optional<ConstantRange> {
@@ -805,7 +805,7 @@ ArgumentUsesSummary collectArgumentUsesPerBlock(Argument &A, Function &F) {
}
auto *I = cast<Instruction>(U);
- bool HasWrite = UpdateUseInfo(I, getArgmentAccessInfo(I, ArgUse, DL));
+ bool HasWrite = UpdateUseInfo(I, getArgumentAccessInfo(I, ArgUse, DL));
Result.HasAnyWrite |= HasWrite;
>From 519b53e65ef6ad5385d80d0726e48bbc3b08992f Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Tue, 18 Feb 2025 17:25:06 +0530
Subject: [PATCH 13/31] [CodeGen][NewPM] Port RegAllocEvictionAdvisor analysis
to NPM (#117309)
Legacy pass used to provide the advisor, so this extracts that logic
into a provider class used by both analysis passes.
All three (Default, Release, Development) legacy passes
`*AdvisorAnalysis` are basically renamed to `*AdvisorProvider`, so the
actual legacy wrapper passes are `*AdvisorAnalysisLegacy`.
There is only one NPM analysis `RegAllocEvictionAnalysis` that switches
between the three providers in the `::run` method, to be cached by the
NPM.
Also adds `RequireAnalysis<RegAllocEvictionAnalysis>` to the optimized
target reg alloc codegen builder.
---
.../llvm}/CodeGen/RegAllocEvictionAdvisor.h | 98 +++++++-
llvm/include/llvm/InitializePasses.h | 2 +-
llvm/include/llvm/Passes/CodeGenPassBuilder.h | 1 +
.../llvm/Passes/MachinePassRegistry.def | 1 +
llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp | 210 +++++++++++-------
llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp | 130 ++++++++---
llvm/lib/CodeGen/RegAllocGreedy.cpp | 13 +-
llvm/lib/CodeGen/RegAllocGreedy.h | 1 -
llvm/lib/CodeGen/RegAllocPriorityAdvisor.h | 2 +-
llvm/lib/Passes/PassBuilder.cpp | 1 +
10 files changed, 330 insertions(+), 129 deletions(-)
rename llvm/{lib => include/llvm}/CodeGen/RegAllocEvictionAdvisor.h (71%)
diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocEvictionAdvisor.h
similarity index 71%
rename from llvm/lib/CodeGen/RegAllocEvictionAdvisor.h
rename to llvm/include/llvm/CodeGen/RegAllocEvictionAdvisor.h
index 52dd946a68540..a14548ff6959e 100644
--- a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h
+++ b/llvm/include/llvm/CodeGen/RegAllocEvictionAdvisor.h
@@ -9,13 +9,18 @@
#ifndef LLVM_CODEGEN_REGALLOCEVICTIONADVISOR_H
#define LLVM_CODEGEN_REGALLOCEVICTIONADVISOR_H
+#include "llvm/ADT/Any.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/Register.h"
#include "llvm/Config/llvm-config.h"
+#include "llvm/IR/PassManager.h"
#include "llvm/MC/MCRegister.h"
#include "llvm/Pass.h"
+#include "llvm/Support/Compiler.h"
namespace llvm {
class AllocationOrder;
@@ -149,6 +154,35 @@ class RegAllocEvictionAdvisor {
const bool EnableLocalReassign;
};
+/// Common provider for legacy and new pass managers.
+/// This keeps the state for logging, and sets up and holds the provider.
+/// The legacy pass itself used to keep the logging state and provider,
+/// so this extraction helps the NPM analysis to reuse the logic.
+/// TODO: Coalesce this with the NPM analysis when legacy PM is removed.
+class RegAllocEvictionAdvisorProvider {
+public:
+ enum class AdvisorMode : int { Default, Release, Development };
+ RegAllocEvictionAdvisorProvider(AdvisorMode Mode, LLVMContext &Ctx)
+ : Ctx(Ctx), Mode(Mode) {}
+
+ virtual ~RegAllocEvictionAdvisorProvider() = default;
+
+ virtual void logRewardIfNeeded(const MachineFunction &MF,
+ llvm::function_ref<float()> GetReward) {}
+
+ virtual std::unique_ptr<RegAllocEvictionAdvisor>
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ MachineBlockFrequencyInfo *MBFI, MachineLoopInfo *Loops) = 0;
+
+ AdvisorMode getAdvisorMode() const { return Mode; }
+
+protected:
+ LLVMContext &Ctx;
+
+private:
+ const AdvisorMode Mode;
+};
+
/// ImmutableAnalysis abstraction for fetching the Eviction Advisor. We model it
/// as an analysis to decouple the user from the implementation insofar as
/// dependencies on other analyses goes. The motivation for it being an
@@ -164,20 +198,20 @@ class RegAllocEvictionAdvisor {
///
/// Because we need to offer additional services in 'development' mode, the
/// implementations of this analysis need to implement RTTI support.
-class RegAllocEvictionAdvisorAnalysis : public ImmutablePass {
+class RegAllocEvictionAdvisorAnalysisLegacy : public ImmutablePass {
public:
enum class AdvisorMode : int { Default, Release, Development };
- RegAllocEvictionAdvisorAnalysis(AdvisorMode Mode)
- : ImmutablePass(ID), Mode(Mode){};
+ RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode Mode)
+ : ImmutablePass(ID), Mode(Mode) {};
static char ID;
/// Get an advisor for the given context (i.e. machine function, etc)
- virtual std::unique_ptr<RegAllocEvictionAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) = 0;
+ RegAllocEvictionAdvisorProvider &getProvider() { return *Provider; }
+
AdvisorMode getAdvisorMode() const { return Mode; }
virtual void logRewardIfNeeded(const MachineFunction &MF,
- llvm::function_ref<float()> GetReward){};
+ function_ref<float()> GetReward) {};
protected:
// This analysis preserves everything, and subclasses may have additional
@@ -185,19 +219,65 @@ class RegAllocEvictionAdvisorAnalysis : public ImmutablePass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
}
+ std::unique_ptr<RegAllocEvictionAdvisorProvider> Provider;
private:
StringRef getPassName() const override;
const AdvisorMode Mode;
};
+/// A MachineFunction analysis for fetching the Eviction Advisor.
+/// This sets up the Provider lazily and caches it.
+/// - in the ML implementation case, the evaluator is stateless but (especially
+/// in the development mode) expensive to set up. With a Module Analysis, we
+/// `require` it and set it up once.
+/// - in the 'development' mode ML case, we want to capture the training log
+/// during allocation (this is a log of features encountered and decisions
+/// made), and then measure a score, potentially a few steps after allocation
+/// completes. So we need a Module analysis to keep the logger state around
+/// until we can make that measurement.
+class RegAllocEvictionAdvisorAnalysis
+ : public AnalysisInfoMixin<RegAllocEvictionAdvisorAnalysis> {
+ static AnalysisKey Key;
+ friend AnalysisInfoMixin<RegAllocEvictionAdvisorAnalysis>;
+
+public:
+ struct Result {
+ // owned by this analysis
+ RegAllocEvictionAdvisorProvider *Provider;
+
+ bool invalidate(MachineFunction &MF, const PreservedAnalyses &PA,
+ MachineFunctionAnalysisManager::Invalidator &Inv) {
+ // Provider is stateless and constructed only once. Do not get
+ // invalidated.
+ return false;
+ }
+ };
+
+ Result run(MachineFunction &MF, MachineFunctionAnalysisManager &MAM);
+
+private:
+ void
+ initializeProvider(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode Mode,
+ LLVMContext &Ctx);
+
+ std::unique_ptr<RegAllocEvictionAdvisorProvider> Provider;
+};
+
/// Specialization for the API used by the analysis infrastructure to create
/// an instance of the eviction advisor.
-template <> Pass *callDefaultCtor<RegAllocEvictionAdvisorAnalysis>();
+template <> Pass *callDefaultCtor<RegAllocEvictionAdvisorAnalysisLegacy>();
+
+RegAllocEvictionAdvisorAnalysisLegacy *createReleaseModeAdvisorAnalysisLegacy();
+
+RegAllocEvictionAdvisorAnalysisLegacy *
+createDevelopmentModeAdvisorAnalysisLegacy();
-RegAllocEvictionAdvisorAnalysis *createReleaseModeAdvisor();
+LLVM_ATTRIBUTE_RETURNS_NONNULL RegAllocEvictionAdvisorProvider *
+createReleaseModeAdvisorProvider(LLVMContext &Ctx);
-RegAllocEvictionAdvisorAnalysis *createDevelopmentModeAdvisor();
+RegAllocEvictionAdvisorProvider *
+createDevelopmentModeAdvisorProvider(LLVMContext &Ctx);
// TODO: move to RegAllocEvictionAdvisor.cpp when we move implementation
// out of RegAllocGreedy.cpp
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index da4ffcd83213a..81a602c8889d8 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -251,7 +251,7 @@ void initializePseudoProbeInserterPass(PassRegistry &);
void initializeRAGreedyPass(PassRegistry &);
void initializeReachingDefAnalysisPass(PassRegistry &);
void initializeReassociateLegacyPassPass(PassRegistry &);
-void initializeRegAllocEvictionAdvisorAnalysisPass(PassRegistry &);
+void initializeRegAllocEvictionAdvisorAnalysisLegacyPass(PassRegistry &);
void initializeRegAllocFastPass(PassRegistry &);
void initializeRegAllocPriorityAdvisorAnalysisPass(PassRegistry &);
void initializeRegAllocScoringPass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index 1458318ff021a..12781e2b84623 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -57,6 +57,7 @@
#include "llvm/CodeGen/PeepholeOptimizer.h"
#include "llvm/CodeGen/PostRASchedulerList.h"
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/RegAllocFast.h"
#include "llvm/CodeGen/RegUsageInfoCollector.h"
#include "llvm/CodeGen/RegUsageInfoPropagate.h"
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 075ebcb829553..2b5e258682585 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -114,6 +114,7 @@ MACHINE_FUNCTION_ANALYSIS("machine-post-dom-tree",
MachinePostDominatorTreeAnalysis())
MACHINE_FUNCTION_ANALYSIS("machine-trace-metrics", MachineTraceMetricsAnalysis())
MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
+MACHINE_FUNCTION_ANALYSIS("regalloc-evict", RegAllocEvictionAdvisorAnalysis())
MACHINE_FUNCTION_ANALYSIS("slot-indexes", SlotIndexesAnalysis())
MACHINE_FUNCTION_ANALYSIS("spill-code-placement", SpillPlacementAnalysis())
MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis())
diff --git a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
index 9656774c6eaae..1a8e11de909e8 100644
--- a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
@@ -11,11 +11,11 @@
//===----------------------------------------------------------------------===//
#include "AllocationOrder.h"
-#include "RegAllocEvictionAdvisor.h"
#include "RegAllocGreedy.h"
#include "llvm/Analysis/InteractiveModelRunner.h"
#include "llvm/Analysis/MLModelRunner.h"
#include "llvm/Analysis/TensorSpec.h"
+#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#if defined(LLVM_HAVE_TF_AOT_REGALLOCEVICTMODEL) || defined(LLVM_HAVE_TFLITE)
#include "llvm/Analysis/ModelUnderTrainingRunner.h"
#include "llvm/Analysis/NoInferenceModelRunner.h"
@@ -115,7 +115,7 @@ class RegAllocScoring : public MachineFunctionPass {
/// RegAllocReward analysis usage.
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
- AU.addRequired<RegAllocEvictionAdvisorAnalysis>();
+ AU.addRequired<RegAllocEvictionAdvisorAnalysisLegacy>();
AU.addRequired<RegAllocPriorityAdvisorAnalysis>();
AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
@@ -389,11 +389,12 @@ class MLEvictAdvisor : public RegAllocEvictionAdvisor {
// ===================================
// Release (AOT) - specifics
// ===================================
-class ReleaseModeEvictionAdvisorAnalysis final
- : public RegAllocEvictionAdvisorAnalysis {
+/// Common provider for legacy and new pass managers.
+class ReleaseModeEvictionAdvisorProvider final
+ : public RegAllocEvictionAdvisorProvider {
public:
- ReleaseModeEvictionAdvisorAnalysis()
- : RegAllocEvictionAdvisorAnalysis(AdvisorMode::Release) {
+ ReleaseModeEvictionAdvisorProvider(LLVMContext &Ctx)
+ : RegAllocEvictionAdvisorProvider(AdvisorMode::Release, Ctx) {
if (EnableDevelopmentFeatures) {
InputFeatures = {RA_EVICT_FEATURES_LIST(
_DECL_FEATURES) RA_EVICT_FIRST_DEVELOPMENT_FEATURE(_DECL_FEATURES)
@@ -403,21 +404,13 @@ class ReleaseModeEvictionAdvisorAnalysis final
}
}
// support for isa<> and dyn_cast.
- static bool classof(const RegAllocEvictionAdvisorAnalysis *R) {
+ static bool classof(const RegAllocEvictionAdvisorProvider *R) {
return R->getAdvisorMode() == AdvisorMode::Release;
}
-private:
- std::vector<TensorSpec> InputFeatures;
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
- AU.addRequired<MachineLoopInfoWrapperPass>();
- RegAllocEvictionAdvisorAnalysis::getAnalysisUsage(AU);
- }
-
std::unique_ptr<RegAllocEvictionAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ MachineBlockFrequencyInfo *MBFI, MachineLoopInfo *Loops) override {
if (!Runner) {
if (InteractiveChannelBaseName.empty())
Runner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
@@ -428,14 +421,45 @@ class ReleaseModeEvictionAdvisorAnalysis final
InteractiveChannelBaseName + ".out",
InteractiveChannelBaseName + ".in");
}
- return std::make_unique<MLEvictAdvisor>(
- MF, RA, Runner.get(),
- getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI(),
- getAnalysis<MachineLoopInfoWrapperPass>().getLI());
+ assert(MBFI && Loops &&
+ "Invalid provider state: must have analysis available");
+ return std::make_unique<MLEvictAdvisor>(MF, RA, Runner.get(), *MBFI,
+ *Loops);
}
+
+private:
+ std::vector<TensorSpec> InputFeatures;
std::unique_ptr<MLModelRunner> Runner;
};
+class ReleaseModeEvictionAdvisorAnalysisLegacy final
+ : public RegAllocEvictionAdvisorAnalysisLegacy {
+public:
+ ReleaseModeEvictionAdvisorAnalysisLegacy()
+ : RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode::Release) {}
+
+ void logRewardIfNeeded(const MachineFunction &MF,
+ llvm::function_ref<float()> GetReward) override {
+ // No-op in release mode
+ }
+
+ bool doInitialization(Module &M) override {
+ Provider =
+ std::make_unique<ReleaseModeEvictionAdvisorProvider>(M.getContext());
+ return false;
+ }
+
+ static bool classof(const RegAllocEvictionAdvisorAnalysisLegacy *R) {
+ return R->getAdvisorMode() == AdvisorMode::Release;
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
+ AU.addRequired<MachineLoopInfoWrapperPass>();
+ RegAllocEvictionAdvisorAnalysisLegacy::getAnalysisUsage(AU);
+ }
+};
+
// ===================================
// Development mode-specifics
// ===================================
@@ -468,11 +492,11 @@ class DevelopmentModeEvictAdvisor : public MLEvictAdvisor {
Logger *const Log;
};
-class DevelopmentModeEvictionAdvisorAnalysis final
- : public RegAllocEvictionAdvisorAnalysis {
+class DevelopmentModeEvictionAdvisorProvider final
+ : public RegAllocEvictionAdvisorProvider {
public:
- DevelopmentModeEvictionAdvisorAnalysis()
- : RegAllocEvictionAdvisorAnalysis(AdvisorMode::Development) {
+ DevelopmentModeEvictionAdvisorProvider(LLVMContext &Ctx)
+ : RegAllocEvictionAdvisorProvider(AdvisorMode::Development, Ctx) {
if (EnableDevelopmentFeatures) {
InputFeatures = {RA_EVICT_FEATURES_LIST(
_DECL_FEATURES) RA_EVICT_FIRST_DEVELOPMENT_FEATURE(_DECL_FEATURES)
@@ -492,44 +516,10 @@ class DevelopmentModeEvictionAdvisorAnalysis final
TensorSpec::createSpec<int32_t>("action_step_type", {1}),
TensorSpec::createSpec<float>("action_reward", {1})};
}
- }
- // support for isa<> and dyn_cast.
- static bool classof(const RegAllocEvictionAdvisorAnalysis *R) {
- return R->getAdvisorMode() == AdvisorMode::Development;
- }
-
- void logRewardIfNeeded(const MachineFunction &MF,
- llvm::function_ref<float()> GetReward) override {
- if (!Log || !Log->hasAnyObservationForContext(MF.getName()))
- return;
- // The function pass manager would run all the function passes for a
- // function, so we assume the last context belongs to this function. If
- // this invariant ever changes, we can implement at that time switching
- // contexts. At this point, it'd be an error
- if (Log->currentContext() != MF.getName()) {
- MF.getFunction().getContext().emitError(
- "The training log context shouldn't have had changed.");
- }
- if (Log->hasObservationInProgress())
- Log->logReward<float>(GetReward());
- }
-
-private:
- std::vector<TensorSpec> InputFeatures;
- std::vector<TensorSpec> TrainingInputFeatures;
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
- AU.addRequired<MachineLoopInfoWrapperPass>();
- RegAllocEvictionAdvisorAnalysis::getAnalysisUsage(AU);
- }
-
- bool doInitialization(Module &M) override {
- LLVMContext &Ctx = M.getContext();
if (ModelUnderTraining.empty() && TrainingLog.empty()) {
Ctx.emitError("Regalloc development mode should be requested with at "
"least logging enabled and/or a training model");
- return false;
+ return;
}
if (ModelUnderTraining.empty())
Runner = std::make_unique<NoInferenceModelRunner>(Ctx, InputFeatures);
@@ -538,15 +528,15 @@ class DevelopmentModeEvictionAdvisorAnalysis final
Ctx, ModelUnderTraining, DecisionName, TrainingInputFeatures);
if (!Runner) {
Ctx.emitError("Regalloc: could not set up the model runner");
- return false;
+ return;
}
if (TrainingLog.empty())
- return false;
+ return;
std::error_code EC;
auto OS = std::make_unique<raw_fd_ostream>(TrainingLog, EC);
if (EC) {
- M.getContext().emitError(EC.message() + ":" + TrainingLog);
- return false;
+ Ctx.emitError(EC.message() + ":" + TrainingLog);
+ return;
}
std::vector<TensorSpec> LFS = InputFeatures;
if (auto *MUTR = dyn_cast<ModelUnderTrainingRunner>(Runner.get()))
@@ -558,25 +548,80 @@ class DevelopmentModeEvictionAdvisorAnalysis final
Log = std::make_unique<Logger>(std::move(OS), LFS, Reward,
/*IncludeReward*/ true);
- return false;
+ return;
+ }
+
+ // support for isa<> and dyn_cast.
+ static bool classof(const RegAllocEvictionAdvisorProvider *R) {
+ return R->getAdvisorMode() == AdvisorMode::Development;
+ }
+
+ void logRewardIfNeeded(const MachineFunction &MF,
+ llvm::function_ref<float()> GetReward) override {
+ if (!Log || !Log->hasAnyObservationForContext(MF.getName()))
+ return;
+ // The function pass manager would run all the function passes for a
+ // function, so we assume the last context belongs to this function. If
+ // this invariant ever changes, we can implement at that time switching
+ // contexts. At this point, it'd be an error
+ if (Log->currentContext() != MF.getName()) {
+ MF.getFunction().getContext().emitError(
+ "The training log context shouldn't have had changed.");
+ }
+ if (Log->hasObservationInProgress())
+ Log->logReward<float>(GetReward());
}
std::unique_ptr<RegAllocEvictionAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ MachineBlockFrequencyInfo *MBFI, MachineLoopInfo *Loops) override {
if (!Runner)
return nullptr;
if (Log)
Log->switchContext(MF.getName());
+ assert(MBFI && Loops &&
+ "Invalid provider state: must have analysis available");
return std::make_unique<DevelopmentModeEvictAdvisor>(
- MF, RA, Runner.get(),
- getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI(),
- getAnalysis<MachineLoopInfoWrapperPass>().getLI(), Log.get());
+ MF, RA, Runner.get(), *MBFI, *Loops, Log.get());
}
+private:
+ std::vector<TensorSpec> InputFeatures;
+ std::vector<TensorSpec> TrainingInputFeatures;
+
std::unique_ptr<MLModelRunner> Runner;
std::unique_ptr<Logger> Log;
};
+class DevelopmentModeEvictionAdvisorAnalysisLegacy final
+ : public RegAllocEvictionAdvisorAnalysisLegacy {
+public:
+ DevelopmentModeEvictionAdvisorAnalysisLegacy()
+ : RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode::Development) {}
+
+ bool doInitialization(Module &M) override {
+ Provider = std::make_unique<DevelopmentModeEvictionAdvisorProvider>(
+ M.getContext());
+ return false;
+ }
+
+ void logRewardIfNeeded(const MachineFunction &MF,
+ llvm::function_ref<float()> GetReward) override {
+ Provider->logRewardIfNeeded(MF, GetReward);
+ }
+
+ // support for isa<> and dyn_cast.
+ static bool classof(const RegAllocEvictionAdvisorAnalysisLegacy *R) {
+ return R->getAdvisorMode() == AdvisorMode::Development;
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
+ AU.addRequired<MachineLoopInfoWrapperPass>();
+ RegAllocEvictionAdvisorAnalysisLegacy::getAnalysisUsage(AU);
+ }
+};
+
#endif // #ifdef LLVM_HAVE_TFLITE
} // namespace
@@ -1127,8 +1172,9 @@ void llvm::extractMBBFrequency(
// Development mode-specific implementations
#ifdef LLVM_HAVE_TFLITE
-RegAllocEvictionAdvisorAnalysis *llvm::createDevelopmentModeAdvisor() {
- return new DevelopmentModeEvictionAdvisorAnalysis();
+RegAllocEvictionAdvisorAnalysisLegacy *
+llvm::createDevelopmentModeAdvisorAnalysisLegacy() {
+ return new DevelopmentModeEvictionAdvisorAnalysisLegacy();
}
int64_t DevelopmentModeEvictAdvisor::tryFindEvictionCandidatePosition(
@@ -1194,18 +1240,32 @@ bool RegAllocScoring::runOnMachineFunction(MachineFunction &MF) {
return *CachedReward;
};
- getAnalysis<RegAllocEvictionAdvisorAnalysis>().logRewardIfNeeded(MF,
- GetReward);
+ getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().logRewardIfNeeded(
+ MF, GetReward);
getAnalysis<RegAllocPriorityAdvisorAnalysis>().logRewardIfNeeded(MF,
GetReward);
return false;
}
#endif // #ifdef LLVM_HAVE_TFLITE
-RegAllocEvictionAdvisorAnalysis *llvm::createReleaseModeAdvisor() {
+RegAllocEvictionAdvisorProvider *
+llvm::createReleaseModeAdvisorProvider(LLVMContext &Ctx) {
+ return new ReleaseModeEvictionAdvisorProvider(Ctx);
+}
+
+RegAllocEvictionAdvisorProvider *
+llvm::createDevelopmentModeAdvisorProvider(LLVMContext &Ctx) {
+#if defined(LLVM_HAVE_TFLITE)
+ return new DevelopmentModeEvictionAdvisorProvider(Ctx);
+#endif
+ return nullptr;
+}
+
+RegAllocEvictionAdvisorAnalysisLegacy *
+llvm::createReleaseModeAdvisorAnalysisLegacy() {
return llvm::isEmbeddedModelEvaluatorValid<CompiledModelType>() ||
!InteractiveChannelBaseName.empty()
- ? new ReleaseModeEvictionAdvisorAnalysis()
+ ? new ReleaseModeEvictionAdvisorAnalysisLegacy()
: nullptr;
}
diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
index a1f441ebd0d5e..2369615ef0fb6 100644
--- a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
@@ -9,12 +9,14 @@
// Implementation of the default eviction advisor and of the Analysis pass.
//
//===----------------------------------------------------------------------===//
-
-#include "RegAllocEvictionAdvisor.h"
+#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "AllocationOrder.h"
#include "RegAllocGreedy.h"
+#include "RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
+#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/IR/Module.h"
@@ -26,17 +28,18 @@
using namespace llvm;
-static cl::opt<RegAllocEvictionAdvisorAnalysis::AdvisorMode> Mode(
+static cl::opt<RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode> Mode(
"regalloc-enable-advisor", cl::Hidden,
- cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default),
+ cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default),
cl::desc("Enable regalloc advisor mode"),
cl::values(
- clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default,
+ clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default,
"default", "Default"),
- clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release,
+ clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release,
"release", "precompiled"),
- clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development,
- "development", "for training")));
+ clEnumValN(
+ RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development,
+ "development", "for training")));
static cl::opt<bool> EnableLocalReassignment(
"enable-local-reassign", cl::Hidden,
@@ -59,59 +62,112 @@ cl::opt<unsigned> EvictInterferenceCutoff(
#define LLVM_HAVE_TF_AOT
#endif
-char RegAllocEvictionAdvisorAnalysis::ID = 0;
-INITIALIZE_PASS(RegAllocEvictionAdvisorAnalysis, "regalloc-evict",
+char RegAllocEvictionAdvisorAnalysisLegacy::ID = 0;
+INITIALIZE_PASS(RegAllocEvictionAdvisorAnalysisLegacy, "regalloc-evict",
"Regalloc eviction policy", false, true)
namespace {
-class DefaultEvictionAdvisorAnalysis final
- : public RegAllocEvictionAdvisorAnalysis {
+class DefaultEvictionAdvisorProvider final
+ : public RegAllocEvictionAdvisorProvider {
public:
- DefaultEvictionAdvisorAnalysis(bool NotAsRequested)
- : RegAllocEvictionAdvisorAnalysis(AdvisorMode::Default),
- NotAsRequested(NotAsRequested) {}
+ DefaultEvictionAdvisorProvider(bool NotAsRequested, LLVMContext &Ctx)
+ : RegAllocEvictionAdvisorProvider(AdvisorMode::Default, Ctx) {
+ if (NotAsRequested)
+ Ctx.emitError("Requested regalloc eviction advisor analysis "
+ "could not be created. Using default");
+ }
// support for isa<> and dyn_cast.
- static bool classof(const RegAllocEvictionAdvisorAnalysis *R) {
+ static bool classof(const RegAllocEvictionAdvisorProvider *R) {
return R->getAdvisorMode() == AdvisorMode::Default;
}
-private:
std::unique_ptr<RegAllocEvictionAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ MachineBlockFrequencyInfo *, MachineLoopInfo *) override {
return std::make_unique<DefaultEvictionAdvisor>(MF, RA);
}
+};
+
+class DefaultEvictionAdvisorAnalysisLegacy final
+ : public RegAllocEvictionAdvisorAnalysisLegacy {
+public:
+ DefaultEvictionAdvisorAnalysisLegacy(bool NotAsRequested)
+ : RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode::Default),
+ NotAsRequested(NotAsRequested) {}
+
bool doInitialization(Module &M) override {
- if (NotAsRequested)
- M.getContext().emitError("Requested regalloc eviction advisor analysis "
- "could not be created. Using default");
- return RegAllocEvictionAdvisorAnalysis::doInitialization(M);
+ Provider.reset(
+ new DefaultEvictionAdvisorProvider(NotAsRequested, M.getContext()));
+ return false;
+ }
+
+ // support for isa<> and dyn_cast.
+ static bool classof(const RegAllocEvictionAdvisorAnalysisLegacy *R) {
+ return R->getAdvisorMode() == AdvisorMode::Default;
}
+
+private:
const bool NotAsRequested;
};
} // namespace
-template <> Pass *llvm::callDefaultCtor<RegAllocEvictionAdvisorAnalysis>() {
- Pass *Ret = nullptr;
+AnalysisKey RegAllocEvictionAdvisorAnalysis::Key;
+
+void RegAllocEvictionAdvisorAnalysis::initializeProvider(
+ RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode Mode, LLVMContext &Ctx) {
+ if (Provider)
+ return;
+ switch (Mode) {
+ case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default:
+ Provider.reset(
+ new DefaultEvictionAdvisorProvider(/*NotAsRequested=*/false, Ctx));
+ return;
+ case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development:
+#if defined(LLVM_HAVE_TFLITE)
+ Provider.reset(createDevelopmentModeAdvisorProvider(Ctx));
+#else
+ Provider.reset(
+ new DefaultEvictionAdvisorProvider(/*NotAsRequested=*/true, Ctx));
+#endif
+ return;
+ case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release:
+ Provider.reset(createReleaseModeAdvisorProvider(Ctx));
+ return;
+ }
+}
+
+RegAllocEvictionAdvisorAnalysis::Result
+RegAllocEvictionAdvisorAnalysis::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ // Lazy initialization of the provider.
+ initializeProvider(::Mode, MF.getFunction().getContext());
+ return Result{Provider.get()};
+}
+
+template <>
+Pass *llvm::callDefaultCtor<RegAllocEvictionAdvisorAnalysisLegacy>() {
switch (Mode) {
- case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default:
- Ret = new DefaultEvictionAdvisorAnalysis(/*NotAsRequested*/ false);
- break;
- case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development:
+ case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default:
+ return new DefaultEvictionAdvisorAnalysisLegacy(/*NotAsRequested=*/false);
+ case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release: {
+ Pass *Ret = createReleaseModeAdvisorAnalysisLegacy();
+ // release mode advisor may not be supported
+ if (Ret)
+ return Ret;
+ return new DefaultEvictionAdvisorAnalysisLegacy(/*NotAsRequested=*/true);
+ }
+ case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development:
#if defined(LLVM_HAVE_TFLITE)
- Ret = createDevelopmentModeAdvisor();
+ return createDevelopmentModeAdvisorAnalysisLegacy();
+#else
+ return new DefaultEvictionAdvisorAnalysisLegacy(/*NotAsRequested=*/true);
#endif
- break;
- case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release:
- Ret = createReleaseModeAdvisor();
- break;
}
- if (Ret)
- return Ret;
- return new DefaultEvictionAdvisorAnalysis(/*NotAsRequested*/ true);
+ llvm_unreachable("unexpected advisor mode");
}
-StringRef RegAllocEvictionAdvisorAnalysis::getPassName() const {
+StringRef RegAllocEvictionAdvisorAnalysisLegacy::getPassName() const {
switch (getAdvisorMode()) {
case AdvisorMode::Default:
return "Default Regalloc Eviction Advisor";
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 7c7eb2ad52b41..9318c1df0b5e2 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -15,7 +15,6 @@
#include "AllocationOrder.h"
#include "InterferenceCache.h"
#include "RegAllocBase.h"
-#include "RegAllocEvictionAdvisor.h"
#include "RegAllocPriorityAdvisor.h"
#include "SplitKit.h"
#include "llvm/ADT/ArrayRef.h"
@@ -46,6 +45,7 @@
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/SlotIndexes.h"
@@ -164,7 +164,7 @@ INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(EdgeBundlesWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(SpillPlacementWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
-INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysis)
+INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysisLegacy)
INITIALIZE_PASS_DEPENDENCY(RegAllocPriorityAdvisorAnalysis)
INITIALIZE_PASS_END(RAGreedy, "greedy",
"Greedy Register Allocator", false, false)
@@ -219,7 +219,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<EdgeBundlesWrapperLegacy>();
AU.addRequired<SpillPlacementWrapperLegacy>();
AU.addRequired<MachineOptimizationRemarkEmitterPass>();
- AU.addRequired<RegAllocEvictionAdvisorAnalysis>();
+ AU.addRequired<RegAllocEvictionAdvisorAnalysisLegacy>();
AU.addRequired<RegAllocPriorityAdvisorAnalysis>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -2765,8 +2765,11 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
: TRI->reverseLocalAssignment();
ExtraInfo.emplace();
- EvictAdvisor =
- getAnalysis<RegAllocEvictionAdvisorAnalysis>().getAdvisor(*MF, *this);
+
+ auto &EvictAdvisorProvider =
+ getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider();
+ EvictAdvisor = EvictAdvisorProvider.getAdvisor(*MF, *this, MBFI, Loops);
+
PriorityAdvisor =
getAnalysis<RegAllocPriorityAdvisorAnalysis>().getAdvisor(*MF, *this);
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index e1ec63b4a5296..1d55a8241d760 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -14,7 +14,6 @@
#include "InterferenceCache.h"
#include "RegAllocBase.h"
-#include "RegAllocEvictionAdvisor.h"
#include "RegAllocPriorityAdvisor.h"
#include "SplitKit.h"
#include "llvm/ADT/ArrayRef.h"
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h
index 32e4598b71539..0758743c2b140 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h
@@ -9,7 +9,7 @@
#ifndef LLVM_CODEGEN_REGALLOCPRIORITYADVISOR_H
#define LLVM_CODEGEN_REGALLOCPRIORITYADVISOR_H
-#include "RegAllocEvictionAdvisor.h"
+#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/Pass.h"
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 8d5c0b3c13e01..96939f89279c6 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -127,6 +127,7 @@
#include "llvm/CodeGen/PeepholeOptimizer.h"
#include "llvm/CodeGen/PostRASchedulerList.h"
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
+#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/RegAllocFast.h"
#include "llvm/CodeGen/RegUsageInfoCollector.h"
#include "llvm/CodeGen/RegUsageInfoPropagate.h"
>From d64f177a2f4ae91cf520111dffed73f3c6b555eb Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg at google.com>
Date: Tue, 18 Feb 2025 12:59:15 +0100
Subject: [PATCH 14/31] [mlir][bazel] Fix `no-allow-shlib-undefined` errors.
The BUILD file changes in https://github.com/llvm/llvm-project/pull/127544 adds `LinalgInterfaces` which is incomplete without `LinalgDialect`.
For now, just add the `LinalgDialect` as dependency to tests which do not otherwise depend on it (but depend on `LinalgInterfaces` through e.g. `TensorDialect`).
This is a temporary solution until the dependency of `TensorDialect` is trimmed to just the `linalg::RelayoutOpInterface`, but not the other linalg interfaces. See https://github.com/llvm/llvm-project/pull/127544#pullrequestreview-2622065243.
---
utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel | 2 ++
1 file changed, 2 insertions(+)
diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
index a55c6f50118dc..d0c9f56f81cb9 100644
--- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
@@ -176,6 +176,7 @@ cc_test(
"//mlir:ArithDialect",
"//mlir:FuncDialect",
"//mlir:IR",
+ "//mlir:LinalgDialect",
"//mlir:Parser",
"//mlir:SCFDialect",
"//mlir:SideEffectInterfaces",
@@ -211,6 +212,7 @@ cc_test(
"//llvm:Support",
"//llvm:TestingSupport",
"//mlir:IR",
+ "//mlir:LinalgDialect",
"//mlir:SPIRVBinaryUtils",
"//mlir:SPIRVDeserialization",
"//mlir:SPIRVDialect",
>From 5ecfe58bab6f55264faa3359085b8623d6dccfa6 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Tue, 3 Dec 2024 10:12:36 +0000
Subject: [PATCH 15/31] [CodeGen][NewPM] Port RegAllocPriorityAdvisor analysis
to NPM
---
.../llvm}/CodeGen/RegAllocPriorityAdvisor.h | 78 +++++++-
llvm/include/llvm/InitializePasses.h | 2 +-
.../llvm/Passes/MachinePassRegistry.def | 1 +
llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp | 6 +-
.../lib/CodeGen/MLRegAllocPriorityAdvisor.cpp | 184 +++++++++++-------
llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp | 2 +-
llvm/lib/CodeGen/RegAllocGreedy.cpp | 9 +-
llvm/lib/CodeGen/RegAllocGreedy.h | 2 +-
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 155 +++++++++++----
llvm/lib/Passes/PassBuilder.cpp | 1 +
10 files changed, 320 insertions(+), 120 deletions(-)
rename llvm/{lib => include/llvm}/CodeGen/RegAllocPriorityAdvisor.h (57%)
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
similarity index 57%
rename from llvm/lib/CodeGen/RegAllocPriorityAdvisor.h
rename to llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
index 0758743c2b140..a53739fdc3fc4 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h
+++ b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
@@ -9,8 +9,10 @@
#ifndef LLVM_CODEGEN_REGALLOCPRIORITYADVISOR_H
#define LLVM_CODEGEN_REGALLOCPRIORITYADVISOR_H
+#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/IR/PassManager.h"
#include "llvm/Pass.h"
namespace llvm {
@@ -68,12 +70,72 @@ class DummyPriorityAdvisor : public RegAllocPriorityAdvisor {
unsigned getPriority(const LiveInterval &LI) const override;
};
-class RegAllocPriorityAdvisorAnalysis : public ImmutablePass {
+/// Common provider for getting the priority advisor and logging rewards.
+/// Legacy analysis forwards all calls to this provider.
+/// New analysis serves the provider as the analysis result.
+/// Expensive setup is done in the constructor, so that the advisor can be
+/// created quickly for every machine function.
+/// TODO: Remove once legacy PM support is dropped.
+class RegAllocPriorityAdvisorProvider {
public:
enum class AdvisorMode : int { Default, Release, Development, Dummy };
- RegAllocPriorityAdvisorAnalysis(AdvisorMode Mode)
- : ImmutablePass(ID), Mode(Mode){};
+ RegAllocPriorityAdvisorProvider(AdvisorMode Mode) : Mode(Mode) {}
+
+ virtual ~RegAllocPriorityAdvisorProvider() = default;
+
+ virtual void logRewardIfNeeded(const MachineFunction &MF,
+ llvm::function_ref<float()> GetReward) {};
+
+ virtual std::unique_ptr<RegAllocPriorityAdvisor>
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA) = 0;
+
+ void setAnalyses(SlotIndexes *SI) { this->SI = SI; }
+
+ AdvisorMode getAdvisorMode() const { return Mode; }
+
+protected:
+ SlotIndexes *SI;
+
+private:
+ const AdvisorMode Mode;
+};
+
+RegAllocPriorityAdvisorProvider *createReleaseModePriorityAdvisorProvider();
+
+RegAllocPriorityAdvisorProvider *
+createDevelopmentModePriorityAdvisorProvider(LLVMContext &Ctx);
+
+class RegAllocPriorityAdvisorAnalysis
+ : public AnalysisInfoMixin<RegAllocPriorityAdvisorAnalysis> {
+ static AnalysisKey Key;
+ friend AnalysisInfoMixin<RegAllocPriorityAdvisorAnalysis>;
+
+public:
+ struct Result {
+ // Owned by this analysis.
+ RegAllocPriorityAdvisorProvider *Provider;
+
+ bool invalidate(MachineFunction &MF, const PreservedAnalyses &PA,
+ MachineFunctionAnalysisManager::Invalidator &Inv) {
+ auto PAC = PA.getChecker<RegAllocPriorityAdvisorAnalysis>();
+ return !PAC.preservedWhenStateless() ||
+ Inv.invalidate<SlotIndexesAnalysis>(MF, PA);
+ }
+ };
+
+ Result run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM);
+
+private:
+ void initializeProvider(LLVMContext &Ctx);
+ std::unique_ptr<RegAllocPriorityAdvisorProvider> Provider;
+};
+
+class RegAllocPriorityAdvisorAnalysisLegacy : public ImmutablePass {
+public:
+ using AdvisorMode = RegAllocPriorityAdvisorProvider::AdvisorMode;
+ RegAllocPriorityAdvisorAnalysisLegacy(AdvisorMode Mode)
+ : ImmutablePass(ID), Mode(Mode) {};
static char ID;
/// Get an advisor for the given context (i.e. machine function, etc)
@@ -81,7 +143,7 @@ class RegAllocPriorityAdvisorAnalysis : public ImmutablePass {
getAdvisor(const MachineFunction &MF, const RAGreedy &RA) = 0;
AdvisorMode getAdvisorMode() const { return Mode; }
virtual void logRewardIfNeeded(const MachineFunction &MF,
- llvm::function_ref<float()> GetReward){};
+ llvm::function_ref<float()> GetReward) {};
protected:
// This analysis preserves everything, and subclasses may have additional
@@ -97,11 +159,13 @@ class RegAllocPriorityAdvisorAnalysis : public ImmutablePass {
/// Specialization for the API used by the analysis infrastructure to create
/// an instance of the priority advisor.
-template <> Pass *callDefaultCtor<RegAllocPriorityAdvisorAnalysis>();
+template <> Pass *callDefaultCtor<RegAllocPriorityAdvisorAnalysisLegacy>();
-RegAllocPriorityAdvisorAnalysis *createReleaseModePriorityAdvisor();
+RegAllocPriorityAdvisorAnalysisLegacy *
+createReleaseModePriorityAdvisorAnalysis();
-RegAllocPriorityAdvisorAnalysis *createDevelopmentModePriorityAdvisor();
+RegAllocPriorityAdvisorAnalysisLegacy *
+createDevelopmentModePriorityAdvisorAnalysis();
} // namespace llvm
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 81a602c8889d8..5b30eb53208a8 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -253,7 +253,7 @@ void initializeReachingDefAnalysisPass(PassRegistry &);
void initializeReassociateLegacyPassPass(PassRegistry &);
void initializeRegAllocEvictionAdvisorAnalysisLegacyPass(PassRegistry &);
void initializeRegAllocFastPass(PassRegistry &);
-void initializeRegAllocPriorityAdvisorAnalysisPass(PassRegistry &);
+void initializeRegAllocPriorityAdvisorAnalysisLegacyPass(PassRegistry &);
void initializeRegAllocScoringPass(PassRegistry &);
void initializeRegBankSelectPass(PassRegistry &);
void initializeRegToMemWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 2b5e258682585..373bd047e2395 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -115,6 +115,7 @@ MACHINE_FUNCTION_ANALYSIS("machine-post-dom-tree",
MACHINE_FUNCTION_ANALYSIS("machine-trace-metrics", MachineTraceMetricsAnalysis())
MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
MACHINE_FUNCTION_ANALYSIS("regalloc-evict", RegAllocEvictionAdvisorAnalysis())
+MACHINE_FUNCTION_ANALYSIS("regalloc-priority", RegAllocPriorityAdvisorAnalysis())
MACHINE_FUNCTION_ANALYSIS("slot-indexes", SlotIndexesAnalysis())
MACHINE_FUNCTION_ANALYSIS("spill-code-placement", SpillPlacementAnalysis())
MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis())
diff --git a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
index 1a8e11de909e8..8215d07b74dba 100644
--- a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp
@@ -116,7 +116,7 @@ class RegAllocScoring : public MachineFunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
AU.addRequired<RegAllocEvictionAdvisorAnalysisLegacy>();
- AU.addRequired<RegAllocPriorityAdvisorAnalysis>();
+ AU.addRequired<RegAllocPriorityAdvisorAnalysisLegacy>();
AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -1242,8 +1242,8 @@ bool RegAllocScoring::runOnMachineFunction(MachineFunction &MF) {
getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().logRewardIfNeeded(
MF, GetReward);
- getAnalysis<RegAllocPriorityAdvisorAnalysis>().logRewardIfNeeded(MF,
- GetReward);
+ getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().logRewardIfNeeded(
+ MF, GetReward);
return false;
}
#endif // #ifdef LLVM_HAVE_TFLITE
diff --git a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
index 9638df81770c1..dfc06eaa0bfe6 100644
--- a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
@@ -12,7 +12,6 @@
#include "AllocationOrder.h"
#include "RegAllocGreedy.h"
-#include "RegAllocPriorityAdvisor.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/InteractiveModelRunner.h"
#include "llvm/Analysis/MLModelRunner.h"
@@ -25,6 +24,7 @@
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/VirtRegMap.h"
@@ -121,23 +121,11 @@ static const std::vector<TensorSpec> InputFeatures{
// ===================================
// Release (AOT) - specifics
// ===================================
-class ReleaseModePriorityAdvisorAnalysis final
- : public RegAllocPriorityAdvisorAnalysis {
+class ReleaseModePriorityAdvisorProvider final
+ : public RegAllocPriorityAdvisorProvider {
public:
- ReleaseModePriorityAdvisorAnalysis()
- : RegAllocPriorityAdvisorAnalysis(AdvisorMode::Release) {}
- // support for isa<> and dyn_cast.
- static bool classof(const RegAllocPriorityAdvisorAnalysis *R) {
- return R->getAdvisorMode() == AdvisorMode::Release;
- }
-
-private:
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesAll();
- AU.addRequired<SlotIndexesWrapperPass>();
- RegAllocPriorityAdvisorAnalysis::getAnalysisUsage(AU);
- }
-
+ ReleaseModePriorityAdvisorProvider()
+ : RegAllocPriorityAdvisorProvider(AdvisorMode::Release) {}
std::unique_ptr<RegAllocPriorityAdvisor>
getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
if (!Runner) {
@@ -150,12 +138,44 @@ class ReleaseModePriorityAdvisorAnalysis final
InteractiveChannelBaseName + ".out",
InteractiveChannelBaseName + ".in");
}
- return std::make_unique<MLPriorityAdvisor>(
- MF, RA, &getAnalysis<SlotIndexesWrapperPass>().getSI(), Runner.get());
+ assert(SI && "SlotIndexes result must be set");
+ return std::make_unique<MLPriorityAdvisor>(MF, RA, SI, Runner.get());
}
+
+private:
std::unique_ptr<MLModelRunner> Runner;
};
+RegAllocPriorityAdvisorProvider *createReleaseModePriorityAdvisorProvider() {
+ return new ReleaseModePriorityAdvisorProvider();
+}
+
+class ReleaseModePriorityAdvisorAnalysisLegacy final
+ : public RegAllocPriorityAdvisorAnalysisLegacy {
+public:
+ ReleaseModePriorityAdvisorAnalysisLegacy()
+ : RegAllocPriorityAdvisorAnalysisLegacy(AdvisorMode::Release) {}
+ // support for isa<> and dyn_cast.
+ static bool classof(const RegAllocPriorityAdvisorAnalysisLegacy *R) {
+ return R->getAdvisorMode() == AdvisorMode::Release;
+ }
+
+private:
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ AU.addRequired<SlotIndexesWrapperPass>();
+ RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
+ }
+
+ std::unique_ptr<RegAllocPriorityAdvisor>
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
+ return Provider->getAdvisor(MF, RA);
+ }
+
+ std::unique_ptr<ReleaseModePriorityAdvisorProvider> Provider;
+};
+
// ===================================
// Development mode-specifics
// ===================================
@@ -186,46 +206,17 @@ class DevelopmentModePriorityAdvisor : public MLPriorityAdvisor {
Logger *const Log;
};
-class DevelopmentModePriorityAdvisorAnalysis final
- : public RegAllocPriorityAdvisorAnalysis {
-public:
- DevelopmentModePriorityAdvisorAnalysis()
- : RegAllocPriorityAdvisorAnalysis(AdvisorMode::Development) {}
- // support for isa<> and dyn_cast.
- static bool classof(const RegAllocPriorityAdvisorAnalysis *R) {
- return R->getAdvisorMode() == AdvisorMode::Development;
- }
-
- void logRewardIfNeeded(const MachineFunction &MF,
- llvm::function_ref<float()> GetReward) override {
- if (!Log || !Log->hasAnyObservationForContext(MF.getName()))
- return;
- // The function pass manager would run all the function passes for a
- // function, so we assume the last context belongs to this function. If
- // this invariant ever changes, we can implement at that time switching
- // contexts. At this point, it'd be an error
- if (Log->currentContext() != MF.getName()) {
- MF.getFunction().getContext().emitError(
- "The training log context shouldn't have had changed.");
- }
- if (Log->hasObservationInProgress())
- Log->logReward<float>(GetReward());
- }
-
-private:
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesAll();
- AU.addRequired<SlotIndexesWrapperPass>();
- RegAllocPriorityAdvisorAnalysis::getAnalysisUsage(AU);
- }
+class DevelopmentModePriorityAdvisorProvider final
+ : public RegAllocPriorityAdvisorProvider {
+public:
// Save all the logs (when requested).
- bool doInitialization(Module &M) override {
- LLVMContext &Ctx = M.getContext();
+ DevelopmentModePriorityAdvisorProvider(LLVMContext &Ctx)
+ : RegAllocPriorityAdvisorProvider(AdvisorMode::Development) {
if (ModelUnderTraining.empty() && TrainingLog.empty()) {
Ctx.emitError("Regalloc development mode should be requested with at "
"least logging enabled and/or a training model");
- return false;
+ return;
}
if (ModelUnderTraining.empty())
Runner = std::make_unique<NoInferenceModelRunner>(Ctx, InputFeatures);
@@ -234,15 +225,15 @@ class DevelopmentModePriorityAdvisorAnalysis final
Ctx, ModelUnderTraining, DecisionName, TrainingInputFeatures);
if (!Runner) {
Ctx.emitError("Regalloc: could not set up the model runner");
- return false;
+ return;
}
if (TrainingLog.empty())
- return false;
+ return;
std::error_code EC;
auto OS = std::make_unique<raw_fd_ostream>(TrainingLog, EC);
if (EC) {
- M.getContext().emitError(EC.message() + ":" + TrainingLog);
- return false;
+ Ctx.emitError(EC.message() + ":" + TrainingLog);
+ return;
}
std::vector<TensorSpec> LFS = InputFeatures;
if (auto *MUTR = dyn_cast<ModelUnderTrainingRunner>(Runner.get()))
@@ -254,7 +245,22 @@ class DevelopmentModePriorityAdvisorAnalysis final
Log = std::make_unique<Logger>(std::move(OS), LFS, Reward,
/*IncludeReward*/ true);
- return false;
+ }
+
+ void logRewardIfNeeded(const MachineFunction &MF,
+ llvm::function_ref<float()> GetReward) override {
+ if (!Log || !Log->hasAnyObservationForContext(MF.getName()))
+ return;
+ // The function pass manager would run all the function passes for a
+ // function, so we assume the last context belongs to this function. If
+ // this invariant ever changes, we can implement at that time switching
+ // contexts. At this point, it'd be an error
+ if (Log->currentContext() != MF.getName()) {
+ MF.getFunction().getContext().emitError(
+ "The training log context shouldn't have had changed.");
+ }
+ if (Log->hasObservationInProgress())
+ Log->logReward<float>(GetReward());
}
std::unique_ptr<RegAllocPriorityAdvisor>
@@ -264,23 +270,68 @@ class DevelopmentModePriorityAdvisorAnalysis final
if (Log) {
Log->switchContext(MF.getName());
}
-
+ assert(SI && "SlotIndexes result must be set");
return std::make_unique<DevelopmentModePriorityAdvisor>(
- MF, RA, &getAnalysis<SlotIndexesWrapperPass>().getSI(), Runner.get(),
- Log.get());
+ MF, RA, SI, Runner.get(), Log.get());
}
std::unique_ptr<MLModelRunner> Runner;
std::unique_ptr<Logger> Log;
};
+
+RegAllocPriorityAdvisorProvider *
+createDevelopmentModePriorityAdvisorProvider(LLVMContext &Ctx) {
+ return new DevelopmentModePriorityAdvisorProvider(Ctx);
+}
+
+class DevelopmentModePriorityAdvisorAnalysisLegacy final
+ : public RegAllocPriorityAdvisorAnalysisLegacy {
+public:
+ DevelopmentModePriorityAdvisorAnalysisLegacy()
+ : RegAllocPriorityAdvisorAnalysisLegacy(AdvisorMode::Development) {}
+
+ // support for isa<> and dyn_cast.
+ static bool classof(const RegAllocPriorityAdvisorAnalysisLegacy *R) {
+ return R->getAdvisorMode() == AdvisorMode::Development;
+ }
+
+ void logRewardIfNeeded(const MachineFunction &MF,
+ llvm::function_ref<float()> GetReward) override {
+ Provider->logRewardIfNeeded(MF, GetReward);
+ }
+
+private:
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ AU.addRequired<SlotIndexesWrapperPass>();
+ RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
+ }
+
+ // Save all the logs (when requested).
+ bool doInitialization(Module &M) override {
+ Provider = std::make_unique<DevelopmentModePriorityAdvisorProvider>(
+ M.getContext());
+ return false;
+ ;
+ }
+
+ std::unique_ptr<RegAllocPriorityAdvisor>
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
+ return Provider->getAdvisor(MF, RA);
+ }
+
+ std::unique_ptr<DevelopmentModePriorityAdvisorProvider> Provider;
+};
#endif //#ifdef LLVM_HAVE_TFLITE
} // namespace llvm
-RegAllocPriorityAdvisorAnalysis *llvm::createReleaseModePriorityAdvisor() {
+RegAllocPriorityAdvisorAnalysisLegacy *
+llvm::createReleaseModePriorityAdvisorAnalysis() {
return llvm::isEmbeddedModelEvaluatorValid<CompiledModelType>() ||
!InteractiveChannelBaseName.empty()
- ? new ReleaseModePriorityAdvisorAnalysis()
+ ? new ReleaseModePriorityAdvisorAnalysisLegacy()
: nullptr;
}
@@ -310,8 +361,9 @@ unsigned MLPriorityAdvisor::getPriority(const LiveInterval &LI) const {
}
#ifdef LLVM_HAVE_TFLITE
-RegAllocPriorityAdvisorAnalysis *llvm::createDevelopmentModePriorityAdvisor() {
- return new DevelopmentModePriorityAdvisorAnalysis();
+RegAllocPriorityAdvisorAnalysisLegacy *
+llvm::createDevelopmentModePriorityAdvisorAnalysis() {
+ return new DevelopmentModePriorityAdvisorAnalysisLegacy();
}
unsigned
diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
index 2369615ef0fb6..30523611977f4 100644
--- a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
@@ -12,11 +12,11 @@
#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "AllocationOrder.h"
#include "RegAllocGreedy.h"
-#include "RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/IR/Module.h"
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 9318c1df0b5e2..16b1c3d73460d 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -15,7 +15,6 @@
#include "AllocationOrder.h"
#include "InterferenceCache.h"
#include "RegAllocBase.h"
-#include "RegAllocPriorityAdvisor.h"
#include "SplitKit.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
@@ -46,6 +45,7 @@
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
+#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/SlotIndexes.h"
@@ -165,7 +165,7 @@ INITIALIZE_PASS_DEPENDENCY(EdgeBundlesWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(SpillPlacementWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysisLegacy)
-INITIALIZE_PASS_DEPENDENCY(RegAllocPriorityAdvisorAnalysis)
+INITIALIZE_PASS_DEPENDENCY(RegAllocPriorityAdvisorAnalysisLegacy)
INITIALIZE_PASS_END(RAGreedy, "greedy",
"Greedy Register Allocator", false, false)
@@ -220,7 +220,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<SpillPlacementWrapperLegacy>();
AU.addRequired<MachineOptimizationRemarkEmitterPass>();
AU.addRequired<RegAllocEvictionAdvisorAnalysisLegacy>();
- AU.addRequired<RegAllocPriorityAdvisorAnalysis>();
+ AU.addRequired<RegAllocPriorityAdvisorAnalysisLegacy>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -2771,7 +2771,8 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
EvictAdvisor = EvictAdvisorProvider.getAdvisor(*MF, *this, MBFI, Loops);
PriorityAdvisor =
- getAnalysis<RegAllocPriorityAdvisorAnalysis>().getAdvisor(*MF, *this);
+ getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().getAdvisor(*MF,
+ *this);
VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI);
SpillerInstance.reset(
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 1d55a8241d760..1698607984bcd 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -14,7 +14,6 @@
#include "InterferenceCache.h"
#include "RegAllocBase.h"
-#include "RegAllocPriorityAdvisor.h"
#include "SplitKit.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
@@ -28,6 +27,7 @@
#include "llvm/CodeGen/LiveRangeEdit.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/SpillPlacement.h"
#include "llvm/CodeGen/Spiller.h"
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index 4525b8fc5a383..0bb1219333237 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -10,7 +10,7 @@
//
//===----------------------------------------------------------------------===//
-#include "RegAllocPriorityAdvisor.h"
+#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "RegAllocGreedy.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/VirtRegMap.h"
@@ -20,72 +20,113 @@
using namespace llvm;
-static cl::opt<RegAllocPriorityAdvisorAnalysis::AdvisorMode> Mode(
+static cl::opt<RegAllocPriorityAdvisorProvider::AdvisorMode> Mode(
"regalloc-enable-priority-advisor", cl::Hidden,
- cl::init(RegAllocPriorityAdvisorAnalysis::AdvisorMode::Default),
+ cl::init(RegAllocPriorityAdvisorProvider::AdvisorMode::Default),
cl::desc("Enable regalloc advisor mode"),
cl::values(
- clEnumValN(RegAllocPriorityAdvisorAnalysis::AdvisorMode::Default,
+ clEnumValN(RegAllocPriorityAdvisorProvider::AdvisorMode::Default,
"default", "Default"),
- clEnumValN(RegAllocPriorityAdvisorAnalysis::AdvisorMode::Release,
+ clEnumValN(RegAllocPriorityAdvisorProvider::AdvisorMode::Release,
"release", "precompiled"),
- clEnumValN(RegAllocPriorityAdvisorAnalysis::AdvisorMode::Development,
+ clEnumValN(RegAllocPriorityAdvisorProvider::AdvisorMode::Development,
"development", "for training"),
clEnumValN(
- RegAllocPriorityAdvisorAnalysis::AdvisorMode::Dummy, "dummy",
+ RegAllocPriorityAdvisorProvider::AdvisorMode::Dummy, "dummy",
"prioritize low virtual register numbers for test and debug")));
-char RegAllocPriorityAdvisorAnalysis::ID = 0;
-INITIALIZE_PASS(RegAllocPriorityAdvisorAnalysis, "regalloc-priority",
+char RegAllocPriorityAdvisorAnalysisLegacy::ID = 0;
+INITIALIZE_PASS(RegAllocPriorityAdvisorAnalysisLegacy, "regalloc-priority",
"Regalloc priority policy", false, true)
namespace {
-class DefaultPriorityAdvisorAnalysis final
- : public RegAllocPriorityAdvisorAnalysis {
+
+class DefaultPriorityAdvisorProvider final
+ : public RegAllocPriorityAdvisorProvider {
+public:
+ DefaultPriorityAdvisorProvider(bool NotAsRequested, LLVMContext &Ctx)
+ : RegAllocPriorityAdvisorProvider(AdvisorMode::Default) {
+ if (NotAsRequested)
+ Ctx.emitError("Requested regalloc priority advisor analysis "
+ "could be created. Using default");
+ }
+
+ // support for isa<> and dyn_cast.
+ static bool classof(const RegAllocPriorityAdvisorProvider *R) {
+ return R->getAdvisorMode() == AdvisorMode::Default;
+ }
+
+ std::unique_ptr<RegAllocPriorityAdvisor>
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ assert(SI && "SlotIndexes result must be set");
+ return std::make_unique<DefaultPriorityAdvisor>(MF, RA, SI);
+ }
+};
+
+class DummyPriorityAdvisorProvider final : public RegAllocPriorityAdvisorProvider {
+public:
+ DummyPriorityAdvisorProvider() : RegAllocPriorityAdvisorProvider(AdvisorMode::Dummy) {}
+
+ static bool classof(const RegAllocPriorityAdvisorProvider *R) {
+ return R->getAdvisorMode() == AdvisorMode::Dummy;
+ }
+
+ std::unique_ptr<RegAllocPriorityAdvisor>
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ assert(SI && "SlotIndexes result must be set");
+ return std::make_unique<DummyPriorityAdvisor>(MF, RA, SI);
+ }
+};
+
+class DefaultPriorityAdvisorAnalysisLegacy final
+ : public RegAllocPriorityAdvisorAnalysisLegacy {
public:
- DefaultPriorityAdvisorAnalysis(bool NotAsRequested)
- : RegAllocPriorityAdvisorAnalysis(AdvisorMode::Default),
+ DefaultPriorityAdvisorAnalysisLegacy(bool NotAsRequested)
+ : RegAllocPriorityAdvisorAnalysisLegacy(AdvisorMode::Default),
NotAsRequested(NotAsRequested) {}
// support for isa<> and dyn_cast.
- static bool classof(const RegAllocPriorityAdvisorAnalysis *R) {
+ static bool classof(const RegAllocPriorityAdvisorAnalysisLegacy *R) {
return R->getAdvisorMode() == AdvisorMode::Default;
}
private:
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<SlotIndexesWrapperPass>();
- RegAllocPriorityAdvisorAnalysis::getAnalysisUsage(AU);
+ RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
+
std::unique_ptr<RegAllocPriorityAdvisor>
getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
- return std::make_unique<DefaultPriorityAdvisor>(
- MF, RA, &getAnalysis<SlotIndexesWrapperPass>().getSI());
+ Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
+ return Provider->getAdvisor(MF, RA);
}
bool doInitialization(Module &M) override {
- if (NotAsRequested)
- M.getContext().emitError("Requested regalloc priority advisor analysis "
- "could be created. Using default");
- return RegAllocPriorityAdvisorAnalysis::doInitialization(M);
+ Provider.reset(
+ new DefaultPriorityAdvisorProvider(NotAsRequested, M.getContext()));
+ return false;
}
+
const bool NotAsRequested;
+ std::unique_ptr<DefaultPriorityAdvisorProvider> Provider;
};
class DummyPriorityAdvisorAnalysis final
- : public RegAllocPriorityAdvisorAnalysis {
+ : public RegAllocPriorityAdvisorAnalysisLegacy {
public:
+ using RegAllocPriorityAdvisorAnalysisLegacy::AdvisorMode;
DummyPriorityAdvisorAnalysis()
- : RegAllocPriorityAdvisorAnalysis(AdvisorMode::Dummy) {}
+ : RegAllocPriorityAdvisorAnalysisLegacy(AdvisorMode::Dummy) {}
// support for isa<> and dyn_cast.
- static bool classof(const RegAllocPriorityAdvisorAnalysis *R) {
+ static bool classof(const RegAllocPriorityAdvisorAnalysisLegacy *R) {
return R->getAdvisorMode() == AdvisorMode::Dummy;
}
private:
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<SlotIndexesWrapperPass>();
- RegAllocPriorityAdvisorAnalysis::getAnalysisUsage(AU);
+ RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
std::unique_ptr<RegAllocPriorityAdvisor>
@@ -97,30 +138,70 @@ class DummyPriorityAdvisorAnalysis final
} // namespace
-template <> Pass *llvm::callDefaultCtor<RegAllocPriorityAdvisorAnalysis>() {
- Pass *Ret = nullptr;
+void RegAllocPriorityAdvisorAnalysis::initializeProvider(LLVMContext &Ctx) {
+ if (Provider)
+ return;
+
switch (Mode) {
- case RegAllocPriorityAdvisorAnalysis::AdvisorMode::Default:
- Ret = new DefaultPriorityAdvisorAnalysis(/*NotAsRequested*/ false);
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Default:
+ Provider.reset(
+ new DefaultPriorityAdvisorProvider(/*NotAsRequested*/ false, Ctx));
break;
- case RegAllocPriorityAdvisorAnalysis::AdvisorMode::Dummy:
- Ret = new DummyPriorityAdvisorAnalysis();
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Dummy:
+ Provider.reset(new DummyPriorityAdvisorProvider());
break;
- case RegAllocPriorityAdvisorAnalysis::AdvisorMode::Development:
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Development:
#if defined(LLVM_HAVE_TFLITE)
- Ret = createDevelopmentModePriorityAdvisor();
+ Provider.reset(createDevelopmentModePriorityAdvisorProvider(Ctx));
#endif
break;
- case RegAllocPriorityAdvisorAnalysis::AdvisorMode::Release:
- Ret = createReleaseModePriorityAdvisor();
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Release:
+ Provider.reset(createReleaseModePriorityAdvisorProvider());
+ break;
+ }
+ if (!Provider)
+ Provider.reset(
+ new DefaultPriorityAdvisorProvider(/*NotAsRequested*/ true, Ctx));
+}
+
+AnalysisKey RegAllocPriorityAdvisorAnalysis::Key;
+
+RegAllocPriorityAdvisorAnalysis::Result
+RegAllocPriorityAdvisorAnalysis::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ // Lazily initialize the provider.
+ initializeProvider(MF.getFunction().getContext());
+ // On each run, update the analysis for the provider.
+ Provider->setAnalyses(&MFAM.getResult<SlotIndexesAnalysis>(MF));
+ // The requiring analysis will construct the advisor.
+ return Result{Provider.get()};
+}
+
+template <>
+Pass *llvm::callDefaultCtor<RegAllocPriorityAdvisorAnalysisLegacy>() {
+ Pass *Ret = nullptr;
+ switch (Mode) {
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Default:
+ Ret = new DefaultPriorityAdvisorAnalysisLegacy(/*NotAsRequested*/ false);
+ break;
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Development:
+#if defined(LLVM_HAVE_TFLITE)
+ Ret = createDevelopmentModePriorityAdvisorAnalysis();
+#endif
+ break;
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Release:
+ Ret = createReleaseModePriorityAdvisorAnalysis();
+ break;
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Dummy:
+ Ret = new DummyPriorityAdvisorAnalysis();
break;
}
if (Ret)
return Ret;
- return new DefaultPriorityAdvisorAnalysis(/*NotAsRequested*/ true);
+ return new DefaultPriorityAdvisorAnalysisLegacy(/*NotAsRequested*/ true);
}
-StringRef RegAllocPriorityAdvisorAnalysis::getPassName() const {
+StringRef RegAllocPriorityAdvisorAnalysisLegacy::getPassName() const {
switch (getAdvisorMode()) {
case AdvisorMode::Default:
return "Default Regalloc Priority Advisor";
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 96939f89279c6..5bb2e7d0abdd9 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -129,6 +129,7 @@
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/RegAllocFast.h"
+#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegUsageInfoCollector.h"
#include "llvm/CodeGen/RegUsageInfoPropagate.h"
#include "llvm/CodeGen/RegisterCoalescerPass.h"
>From 80d4f3e19d249907b97fc822ea6c3440354ed02c Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Wed, 11 Dec 2024 09:03:53 +0000
Subject: [PATCH 16/31] use the provider as the analysis result
Avoid setting the advisor from the legacy wrapper after setting all
other analyses.
---
.../llvm/CodeGen/RegAllocPriorityAdvisor.h | 5 +++--
.../lib/CodeGen/MLRegAllocPriorityAdvisor.cpp | 19 ++++++++++---------
llvm/lib/CodeGen/RegAllocGreedy.cpp | 6 +++---
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 8 ++++----
4 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
index a53739fdc3fc4..600e63110ea91 100644
--- a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
+++ b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
@@ -139,8 +139,7 @@ class RegAllocPriorityAdvisorAnalysisLegacy : public ImmutablePass {
static char ID;
/// Get an advisor for the given context (i.e. machine function, etc)
- virtual std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) = 0;
+ virtual std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() = 0;
AdvisorMode getAdvisorMode() const { return Mode; }
virtual void logRewardIfNeeded(const MachineFunction &MF,
llvm::function_ref<float()> GetReward) {};
@@ -152,6 +151,8 @@ class RegAllocPriorityAdvisorAnalysisLegacy : public ImmutablePass {
AU.setPreservesAll();
}
+ std::unique_ptr<RegAllocPriorityAdvisorProvider> Provider;
+
private:
StringRef getPassName() const override;
const AdvisorMode Mode;
diff --git a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
index dfc06eaa0bfe6..dd598767f1bbc 100644
--- a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
@@ -167,13 +167,17 @@ class ReleaseModePriorityAdvisorAnalysisLegacy final
RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
- std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() override {
Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
- return Provider->getAdvisor(MF, RA);
+ return Provider;
}
- std::unique_ptr<ReleaseModePriorityAdvisorProvider> Provider;
+ bool doInitialization(Module &M) override {
+ Provider = std::make_unique<ReleaseModePriorityAdvisorProvider>();
+ return false;
+ }
+
+ // std::unique_ptr<ReleaseModePriorityAdvisorProvider> Provider;
};
// ===================================
@@ -315,13 +319,10 @@ class DevelopmentModePriorityAdvisorAnalysisLegacy final
;
}
- std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() override {
Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
- return Provider->getAdvisor(MF, RA);
+ return Provider;
}
-
- std::unique_ptr<DevelopmentModePriorityAdvisorProvider> Provider;
};
#endif //#ifdef LLVM_HAVE_TFLITE
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 16b1c3d73460d..3a957df85a8e6 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2770,9 +2770,9 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider();
EvictAdvisor = EvictAdvisorProvider.getAdvisor(*MF, *this, MBFI, Loops);
- PriorityAdvisor =
- getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().getAdvisor(*MF,
- *this);
+ PriorityAdvisor = getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>()
+ .getProvider()
+ ->getAdvisor(*MF, *this);
VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI);
SpillerInstance.reset(
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index 0bb1219333237..c66c9b1fd06d6 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -96,11 +96,11 @@ class DefaultPriorityAdvisorAnalysisLegacy final
RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
- std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() override {
Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
- return Provider->getAdvisor(MF, RA);
+ return Provider;
}
+
bool doInitialization(Module &M) override {
Provider.reset(
new DefaultPriorityAdvisorProvider(NotAsRequested, M.getContext()));
@@ -108,7 +108,7 @@ class DefaultPriorityAdvisorAnalysisLegacy final
}
const bool NotAsRequested;
- std::unique_ptr<DefaultPriorityAdvisorProvider> Provider;
+ // std::unique_ptr<DefaultPriorityAdvisorProvider> Provider;
};
class DummyPriorityAdvisorAnalysis final
>From 98b9fc743649a6d7c0196037153304d470c94973 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Thu, 26 Dec 2024 09:31:53 +0000
Subject: [PATCH 17/31] Suggestions and remove createProvider methods
---
.../llvm/CodeGen/RegAllocPriorityAdvisor.h | 7 +---
.../lib/CodeGen/MLRegAllocPriorityAdvisor.cpp | 29 +++++++++------
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 37 ++++++++-----------
3 files changed, 35 insertions(+), 38 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
index 600e63110ea91..bc00fb0c85a02 100644
--- a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
+++ b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
@@ -101,11 +101,6 @@ class RegAllocPriorityAdvisorProvider {
const AdvisorMode Mode;
};
-RegAllocPriorityAdvisorProvider *createReleaseModePriorityAdvisorProvider();
-
-RegAllocPriorityAdvisorProvider *
-createDevelopmentModePriorityAdvisorProvider(LLVMContext &Ctx);
-
class RegAllocPriorityAdvisorAnalysis
: public AnalysisInfoMixin<RegAllocPriorityAdvisorAnalysis> {
static AnalysisKey Key;
@@ -128,6 +123,8 @@ class RegAllocPriorityAdvisorAnalysis
private:
void initializeProvider(LLVMContext &Ctx);
+ void initializeMLProvider(RegAllocPriorityAdvisorProvider::AdvisorMode Mode,
+ LLVMContext &Ctx);
std::unique_ptr<RegAllocPriorityAdvisorProvider> Provider;
};
diff --git a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
index dd598767f1bbc..dedd20b31d1c0 100644
--- a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
@@ -146,10 +146,6 @@ class ReleaseModePriorityAdvisorProvider final
std::unique_ptr<MLModelRunner> Runner;
};
-RegAllocPriorityAdvisorProvider *createReleaseModePriorityAdvisorProvider() {
- return new ReleaseModePriorityAdvisorProvider();
-}
-
class ReleaseModePriorityAdvisorAnalysisLegacy final
: public RegAllocPriorityAdvisorAnalysisLegacy {
public:
@@ -176,8 +172,6 @@ class ReleaseModePriorityAdvisorAnalysisLegacy final
Provider = std::make_unique<ReleaseModePriorityAdvisorProvider>();
return false;
}
-
- // std::unique_ptr<ReleaseModePriorityAdvisorProvider> Provider;
};
// ===================================
@@ -283,11 +277,6 @@ class DevelopmentModePriorityAdvisorProvider final
std::unique_ptr<Logger> Log;
};
-RegAllocPriorityAdvisorProvider *
-createDevelopmentModePriorityAdvisorProvider(LLVMContext &Ctx) {
- return new DevelopmentModePriorityAdvisorProvider(Ctx);
-}
-
class DevelopmentModePriorityAdvisorAnalysisLegacy final
: public RegAllocPriorityAdvisorAnalysisLegacy {
public:
@@ -410,3 +399,21 @@ DevelopmentModePriorityAdvisor::getPriority(const LiveInterval &LI) const {
}
#endif // #ifdef LLVM_HAVE_TFLITE
+
+void RegAllocPriorityAdvisorAnalysis::initializeMLProvider(
+ RegAllocPriorityAdvisorProvider::AdvisorMode Mode, LLVMContext &Ctx) {
+ if (Provider)
+ return;
+ switch (Mode) {
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Development:
+#if defined(LLVM_HAVE_TFLITE)
+ Provider.reset(new DevelopmentModePriorityAdvisorProvider(Ctx));
+#endif
+ break;
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Release:
+ Provider.reset(new ReleaseModePriorityAdvisorProvider());
+ break;
+ default:
+ break;
+ }
+}
\ No newline at end of file
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index c66c9b1fd06d6..d9a26186aca25 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -129,10 +129,14 @@ class DummyPriorityAdvisorAnalysis final
RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
- std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
- return std::make_unique<DummyPriorityAdvisor>(
- MF, RA, &getAnalysis<SlotIndexesWrapperPass>().getSI());
+ std::unique_ptr<RegAllocPriorityAdvisorProvider>&
+ getProvider() override {
+ Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
+ return Provider;
+ }
+ bool doInitialization(Module &M) override {
+ Provider.reset(new DummyPriorityAdvisorProvider());
+ return false;
}
};
@@ -141,27 +145,16 @@ class DummyPriorityAdvisorAnalysis final
void RegAllocPriorityAdvisorAnalysis::initializeProvider(LLVMContext &Ctx) {
if (Provider)
return;
-
- switch (Mode) {
- case RegAllocPriorityAdvisorProvider::AdvisorMode::Default:
- Provider.reset(
- new DefaultPriorityAdvisorProvider(/*NotAsRequested*/ false, Ctx));
- break;
- case RegAllocPriorityAdvisorProvider::AdvisorMode::Dummy:
+ if (Mode == RegAllocPriorityAdvisorProvider::AdvisorMode::Dummy)
Provider.reset(new DummyPriorityAdvisorProvider());
- break;
- case RegAllocPriorityAdvisorProvider::AdvisorMode::Development:
-#if defined(LLVM_HAVE_TFLITE)
- Provider.reset(createDevelopmentModePriorityAdvisorProvider(Ctx));
-#endif
- break;
- case RegAllocPriorityAdvisorProvider::AdvisorMode::Release:
- Provider.reset(createReleaseModePriorityAdvisorProvider());
- break;
- }
+ else if (Mode == RegAllocPriorityAdvisorProvider::AdvisorMode::Default)
+ Provider.reset(
+ new DefaultPriorityAdvisorProvider(/*NotAsRequested=*/false, Ctx));
+ else
+ initializeMLProvider(Mode, Ctx);
if (!Provider)
Provider.reset(
- new DefaultPriorityAdvisorProvider(/*NotAsRequested*/ true, Ctx));
+ new DefaultPriorityAdvisorProvider(/*NotAsRequested=*/true, Ctx));
}
AnalysisKey RegAllocPriorityAdvisorAnalysis::Key;
>From 0f03d4da8782e848a4985859ee66016e90134164 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Thu, 26 Dec 2024 10:13:51 +0000
Subject: [PATCH 18/31] Apply suggestions and remove setAnalyses
---
.../llvm/CodeGen/RegAllocPriorityAdvisor.h | 12 ++++--------
.../lib/CodeGen/MLRegAllocPriorityAdvisor.cpp | 16 ++++------------
llvm/lib/CodeGen/RegAllocGreedy.cpp | 2 +-
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 19 ++++---------------
4 files changed, 13 insertions(+), 36 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
index bc00fb0c85a02..82232eb4732a3 100644
--- a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
+++ b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
@@ -85,18 +85,14 @@ class RegAllocPriorityAdvisorProvider {
virtual ~RegAllocPriorityAdvisorProvider() = default;
virtual void logRewardIfNeeded(const MachineFunction &MF,
- llvm::function_ref<float()> GetReward) {};
+ function_ref<float()> GetReward) {};
virtual std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) = 0;
-
- void setAnalyses(SlotIndexes *SI) { this->SI = SI; }
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ SlotIndexes *SI) = 0;
AdvisorMode getAdvisorMode() const { return Mode; }
-protected:
- SlotIndexes *SI;
-
private:
const AdvisorMode Mode;
};
@@ -136,7 +132,7 @@ class RegAllocPriorityAdvisorAnalysisLegacy : public ImmutablePass {
static char ID;
/// Get an advisor for the given context (i.e. machine function, etc)
- virtual std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() = 0;
+ RegAllocPriorityAdvisorProvider &getProvider() { return *Provider; }
AdvisorMode getAdvisorMode() const { return Mode; }
virtual void logRewardIfNeeded(const MachineFunction &MF,
llvm::function_ref<float()> GetReward) {};
diff --git a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
index dedd20b31d1c0..8b195b36a711f 100644
--- a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
@@ -127,7 +127,8 @@ class ReleaseModePriorityAdvisorProvider final
ReleaseModePriorityAdvisorProvider()
: RegAllocPriorityAdvisorProvider(AdvisorMode::Release) {}
std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ SlotIndexes *SI) override {
if (!Runner) {
if (InteractiveChannelBaseName.empty())
Runner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
@@ -163,11 +164,6 @@ class ReleaseModePriorityAdvisorAnalysisLegacy final
RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
- std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() override {
- Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
- return Provider;
- }
-
bool doInitialization(Module &M) override {
Provider = std::make_unique<ReleaseModePriorityAdvisorProvider>();
return false;
@@ -262,7 +258,8 @@ class DevelopmentModePriorityAdvisorProvider final
}
std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ SlotIndexes *SI) override {
if (!Runner)
return nullptr;
if (Log) {
@@ -307,11 +304,6 @@ class DevelopmentModePriorityAdvisorAnalysisLegacy final
return false;
;
}
-
- std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() override {
- Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
- return Provider;
- }
};
#endif //#ifdef LLVM_HAVE_TFLITE
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 3a957df85a8e6..e31172cee4021 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2772,7 +2772,7 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
PriorityAdvisor = getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>()
.getProvider()
- ->getAdvisor(*MF, *this);
+ .getAdvisor(*MF, *this, Indexes);
VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI);
SpillerInstance.reset(
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index d9a26186aca25..97cd85eac6129 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -57,7 +57,8 @@ class DefaultPriorityAdvisorProvider final
}
std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ SlotIndexes *SI) override {
assert(SI && "SlotIndexes result must be set");
return std::make_unique<DefaultPriorityAdvisor>(MF, RA, SI);
}
@@ -72,7 +73,8 @@ class DummyPriorityAdvisorProvider final : public RegAllocPriorityAdvisorProvide
}
std::unique_ptr<RegAllocPriorityAdvisor>
- getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override {
+ getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
+ SlotIndexes *SI) override {
assert(SI && "SlotIndexes result must be set");
return std::make_unique<DummyPriorityAdvisor>(MF, RA, SI);
}
@@ -96,11 +98,6 @@ class DefaultPriorityAdvisorAnalysisLegacy final
RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
- std::unique_ptr<RegAllocPriorityAdvisorProvider> &getProvider() override {
- Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
- return Provider;
- }
-
bool doInitialization(Module &M) override {
Provider.reset(
new DefaultPriorityAdvisorProvider(NotAsRequested, M.getContext()));
@@ -108,7 +105,6 @@ class DefaultPriorityAdvisorAnalysisLegacy final
}
const bool NotAsRequested;
- // std::unique_ptr<DefaultPriorityAdvisorProvider> Provider;
};
class DummyPriorityAdvisorAnalysis final
@@ -129,11 +125,6 @@ class DummyPriorityAdvisorAnalysis final
RegAllocPriorityAdvisorAnalysisLegacy::getAnalysisUsage(AU);
}
- std::unique_ptr<RegAllocPriorityAdvisorProvider>&
- getProvider() override {
- Provider->setAnalyses(&getAnalysis<SlotIndexesWrapperPass>().getSI());
- return Provider;
- }
bool doInitialization(Module &M) override {
Provider.reset(new DummyPriorityAdvisorProvider());
return false;
@@ -164,8 +155,6 @@ RegAllocPriorityAdvisorAnalysis::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
// Lazily initialize the provider.
initializeProvider(MF.getFunction().getContext());
- // On each run, update the analysis for the provider.
- Provider->setAnalyses(&MFAM.getResult<SlotIndexesAnalysis>(MF));
// The requiring analysis will construct the advisor.
return Result{Provider.get()};
}
>From c0d8f16871656d0cbb172f23d593fba530cb996c Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Tue, 7 Jan 2025 09:57:23 +0000
Subject: [PATCH 19/31] run clang format
---
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index 97cd85eac6129..e7cec6661bf77 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -64,9 +64,11 @@ class DefaultPriorityAdvisorProvider final
}
};
-class DummyPriorityAdvisorProvider final : public RegAllocPriorityAdvisorProvider {
+class DummyPriorityAdvisorProvider final
+ : public RegAllocPriorityAdvisorProvider {
public:
- DummyPriorityAdvisorProvider() : RegAllocPriorityAdvisorProvider(AdvisorMode::Dummy) {}
+ DummyPriorityAdvisorProvider()
+ : RegAllocPriorityAdvisorProvider(AdvisorMode::Dummy) {}
static bool classof(const RegAllocPriorityAdvisorProvider *R) {
return R->getAdvisorMode() == AdvisorMode::Dummy;
>From ff1aae3d38b11cc29e05f157b9d923112c5f9b98 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Thu, 23 Jan 2025 09:10:27 +0000
Subject: [PATCH 20/31] Add newline at EOF
---
llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
index 8b195b36a711f..20ae298d627aa 100644
--- a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
@@ -408,4 +408,4 @@ void RegAllocPriorityAdvisorAnalysis::initializeMLProvider(
default:
break;
}
-}
\ No newline at end of file
+}
>From 547ecdb036606239e3a9be68cbb4652dc6663cfa Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Tue, 4 Feb 2025 06:57:18 +0000
Subject: [PATCH 21/31] make SI a reference
---
llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h | 2 +-
llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp | 10 ++++------
llvm/lib/CodeGen/RegAllocGreedy.cpp | 2 +-
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 10 ++++------
4 files changed, 10 insertions(+), 14 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
index 82232eb4732a3..974205e8a504c 100644
--- a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
+++ b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
@@ -89,7 +89,7 @@ class RegAllocPriorityAdvisorProvider {
virtual std::unique_ptr<RegAllocPriorityAdvisor>
getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
- SlotIndexes *SI) = 0;
+ SlotIndexes &SI) = 0;
AdvisorMode getAdvisorMode() const { return Mode; }
diff --git a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
index 20ae298d627aa..4f15db5645901 100644
--- a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
@@ -128,7 +128,7 @@ class ReleaseModePriorityAdvisorProvider final
: RegAllocPriorityAdvisorProvider(AdvisorMode::Release) {}
std::unique_ptr<RegAllocPriorityAdvisor>
getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
- SlotIndexes *SI) override {
+ SlotIndexes &SI) override {
if (!Runner) {
if (InteractiveChannelBaseName.empty())
Runner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
@@ -139,8 +139,7 @@ class ReleaseModePriorityAdvisorProvider final
InteractiveChannelBaseName + ".out",
InteractiveChannelBaseName + ".in");
}
- assert(SI && "SlotIndexes result must be set");
- return std::make_unique<MLPriorityAdvisor>(MF, RA, SI, Runner.get());
+ return std::make_unique<MLPriorityAdvisor>(MF, RA, &SI, Runner.get());
}
private:
@@ -259,15 +258,14 @@ class DevelopmentModePriorityAdvisorProvider final
std::unique_ptr<RegAllocPriorityAdvisor>
getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
- SlotIndexes *SI) override {
+ SlotIndexes &SI) override {
if (!Runner)
return nullptr;
if (Log) {
Log->switchContext(MF.getName());
}
- assert(SI && "SlotIndexes result must be set");
return std::make_unique<DevelopmentModePriorityAdvisor>(
- MF, RA, SI, Runner.get(), Log.get());
+ MF, RA, &SI, Runner.get(), Log.get());
}
std::unique_ptr<MLModelRunner> Runner;
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index e31172cee4021..bd81d630f9d1f 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -2772,7 +2772,7 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
PriorityAdvisor = getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>()
.getProvider()
- .getAdvisor(*MF, *this, Indexes);
+ .getAdvisor(*MF, *this, *Indexes);
VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI);
SpillerInstance.reset(
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index e7cec6661bf77..cbc443127cc0d 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -58,9 +58,8 @@ class DefaultPriorityAdvisorProvider final
std::unique_ptr<RegAllocPriorityAdvisor>
getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
- SlotIndexes *SI) override {
- assert(SI && "SlotIndexes result must be set");
- return std::make_unique<DefaultPriorityAdvisor>(MF, RA, SI);
+ SlotIndexes &SI) override {
+ return std::make_unique<DefaultPriorityAdvisor>(MF, RA, &SI);
}
};
@@ -76,9 +75,8 @@ class DummyPriorityAdvisorProvider final
std::unique_ptr<RegAllocPriorityAdvisor>
getAdvisor(const MachineFunction &MF, const RAGreedy &RA,
- SlotIndexes *SI) override {
- assert(SI && "SlotIndexes result must be set");
- return std::make_unique<DummyPriorityAdvisor>(MF, RA, SI);
+ SlotIndexes &SI) override {
+ return std::make_unique<DummyPriorityAdvisor>(MF, RA, &SI);
}
};
>From 3c50c9db5803b61fef45e6e1f55da44208b03c7d Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Tue, 4 Feb 2025 09:18:49 +0000
Subject: [PATCH 22/31] AS, remove initializeMLProvider()
---
.../llvm/CodeGen/RegAllocPriorityAdvisor.h | 6 +++++
.../lib/CodeGen/MLRegAllocPriorityAdvisor.cpp | 24 +++++++------------
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 21 ++++++++++++----
3 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
index 974205e8a504c..b0355795370d8 100644
--- a/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
+++ b/llvm/include/llvm/CodeGen/RegAllocPriorityAdvisor.h
@@ -161,6 +161,12 @@ createReleaseModePriorityAdvisorAnalysis();
RegAllocPriorityAdvisorAnalysisLegacy *
createDevelopmentModePriorityAdvisorAnalysis();
+LLVM_ATTRIBUTE_RETURNS_NONNULL RegAllocPriorityAdvisorProvider *
+createReleaseModePriorityAdvisorProvider();
+
+LLVM_ATTRIBUTE_RETURNS_NONNULL RegAllocPriorityAdvisorProvider *
+createDevelopmentModePriorityAdvisorProvider(LLVMContext &Ctx);
+
} // namespace llvm
#endif // LLVM_CODEGEN_REGALLOCPRIORITYADVISOR_H
diff --git a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
index 4f15db5645901..50ecd3ce2c88c 100644
--- a/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/MLRegAllocPriorityAdvisor.cpp
@@ -388,22 +388,14 @@ DevelopmentModePriorityAdvisor::getPriority(const LiveInterval &LI) const {
return static_cast<unsigned>(Prio);
}
+RegAllocPriorityAdvisorProvider *
+llvm::createDevelopmentModePriorityAdvisorProvider(LLVMContext &Ctx) {
+ return new DevelopmentModePriorityAdvisorProvider(Ctx);
+}
+
#endif // #ifdef LLVM_HAVE_TFLITE
-void RegAllocPriorityAdvisorAnalysis::initializeMLProvider(
- RegAllocPriorityAdvisorProvider::AdvisorMode Mode, LLVMContext &Ctx) {
- if (Provider)
- return;
- switch (Mode) {
- case RegAllocPriorityAdvisorProvider::AdvisorMode::Development:
-#if defined(LLVM_HAVE_TFLITE)
- Provider.reset(new DevelopmentModePriorityAdvisorProvider(Ctx));
-#endif
- break;
- case RegAllocPriorityAdvisorProvider::AdvisorMode::Release:
- Provider.reset(new ReleaseModePriorityAdvisorProvider());
- break;
- default:
- break;
- }
+RegAllocPriorityAdvisorProvider *
+llvm::createReleaseModePriorityAdvisorProvider() {
+ return new ReleaseModePriorityAdvisorProvider();
}
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index cbc443127cc0d..78a0d5ba0c763 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -136,16 +136,27 @@ class DummyPriorityAdvisorAnalysis final
void RegAllocPriorityAdvisorAnalysis::initializeProvider(LLVMContext &Ctx) {
if (Provider)
return;
- if (Mode == RegAllocPriorityAdvisorProvider::AdvisorMode::Dummy)
+ switch (Mode) {
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Dummy:
Provider.reset(new DummyPriorityAdvisorProvider());
- else if (Mode == RegAllocPriorityAdvisorProvider::AdvisorMode::Default)
+ return;
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Default:
Provider.reset(
new DefaultPriorityAdvisorProvider(/*NotAsRequested=*/false, Ctx));
- else
- initializeMLProvider(Mode, Ctx);
- if (!Provider)
+ return;
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Development:
+#if defined(LLVM_HAVE_TFLITE)
+ Provider.reset(createDevelopmentModePriorityAdvisorProvider(Ctx));
+#else
Provider.reset(
new DefaultPriorityAdvisorProvider(/*NotAsRequested=*/true, Ctx));
+#endif
+ assert(Provider && "PriorityAdvisorProvider cannot be null");
+ return;
+ case RegAllocPriorityAdvisorProvider::AdvisorMode::Release:
+ Provider.reset(createReleaseModePriorityAdvisorProvider());
+ return;
+ }
}
AnalysisKey RegAllocPriorityAdvisorAnalysis::Key;
>From 9fa11b9885b40f1db9431c6eb97e9febbc5cf697 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Tue, 4 Feb 2025 10:20:27 +0000
Subject: [PATCH 23/31] remove assert
---
llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
index 78a0d5ba0c763..544ff72a04efd 100644
--- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.cpp
@@ -151,7 +151,6 @@ void RegAllocPriorityAdvisorAnalysis::initializeProvider(LLVMContext &Ctx) {
Provider.reset(
new DefaultPriorityAdvisorProvider(/*NotAsRequested=*/true, Ctx));
#endif
- assert(Provider && "PriorityAdvisorProvider cannot be null");
return;
case RegAllocPriorityAdvisorProvider::AdvisorMode::Release:
Provider.reset(createReleaseModePriorityAdvisorProvider());
>From 575801cc0e3753902cd88af58a6ab7a0e86a3abe Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Wed, 11 Dec 2024 08:51:55 +0000
Subject: [PATCH 24/31] [CodeGen][NewPM] Port RegAllocGreedy to NPM
---
llvm/include/llvm/CodeGen/MachineFunction.h | 1 +
llvm/include/llvm/CodeGen/Passes.h | 2 +-
llvm/include/llvm/InitializePasses.h | 2 +-
.../llvm/Passes/MachinePassRegistry.def | 9 +
llvm/lib/CodeGen/CodeGen.cpp | 2 +-
llvm/lib/CodeGen/RegAllocGreedy.cpp | 185 ++++++++++++++----
llvm/lib/CodeGen/RegAllocGreedy.h | 57 +++---
llvm/lib/Passes/PassBuilder.cpp | 1 +
8 files changed, 196 insertions(+), 63 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index f1e595cde54e3..7fd0994883fe8 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -927,6 +927,7 @@ class LLVM_ABI MachineFunction {
/// Run the current MachineFunction through the machine code verifier, useful
/// for debugger use.
+ /// TODO: Add the param LiveStks
/// \returns true if no problems were found.
bool verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
const char *Banner = nullptr, raw_ostream *OS = nullptr,
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index b5d2a7e6bf035..0182f21bee5f5 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -171,7 +171,7 @@ namespace llvm {
extern char &LiveRangeShrinkID;
/// Greedy register allocator.
- extern char &RAGreedyID;
+ extern char &RAGreedyLegacyID;
/// Basic register allocator.
extern char &RABasicID;
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 5b30eb53208a8..69c9e14541907 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -248,7 +248,7 @@ void initializeProfileSummaryInfoWrapperPassPass(PassRegistry &);
void initializePromoteLegacyPassPass(PassRegistry &);
void initializeRABasicPass(PassRegistry &);
void initializePseudoProbeInserterPass(PassRegistry &);
-void initializeRAGreedyPass(PassRegistry &);
+void initializeRAGreedyLegacyPass(PassRegistry &);
void initializeReachingDefAnalysisPass(PassRegistry &);
void initializeReassociateLegacyPassPass(PassRegistry &);
void initializeRegAllocEvictionAdvisorAnalysisLegacyPass(PassRegistry &);
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 373bd047e2395..78b4c8153e26b 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -194,6 +194,15 @@ MACHINE_FUNCTION_PASS_WITH_PARAMS(
return parseRegAllocFastPassOptions(*PB, Params);
},
"filter=reg-filter;no-clear-vregs")
+
+MACHINE_FUNCTION_PASS_WITH_PARAMS(
+ "regallocgreedy", "RAGreedy",
+ [](RegAllocFilterFunc F) { return RAGreedyPass(F); },
+ [PB = this](StringRef Params) {
+ // TODO: parseRegAllocFilter(*PB, Params);
+ return Expected<RegAllocFilterFunc>(nullptr);
+ }, ""
+)
#undef MACHINE_FUNCTION_PASS_WITH_PARAMS
// After a pass is converted to new pass manager, its entry should be moved from
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 35df2a479a545..21f76bdb2ad6b 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -112,7 +112,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
initializePreISelIntrinsicLoweringLegacyPassPass(Registry);
initializeProcessImplicitDefsPass(Registry);
initializeRABasicPass(Registry);
- initializeRAGreedyPass(Registry);
+ initializeRAGreedyLegacyPass(Registry);
initializeRegAllocFastPass(Registry);
initializeRegUsageInfoCollectorLegacyPass(Registry);
initializeRegUsageInfoPropagationLegacyPass(Registry);
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index bd81d630f9d1f..f4cc80c751350 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -43,8 +43,10 @@
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
+#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
+#include "llvm/CodeGen/RegAllocGreedyPass.h"
#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
@@ -55,6 +57,7 @@
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/IR/Analysis.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/LLVMContext.h"
@@ -146,11 +149,134 @@ static cl::opt<unsigned> SplitThresholdForRegWithHint(
static RegisterRegAlloc greedyRegAlloc("greedy", "greedy register allocator",
createGreedyRegisterAllocator);
-char RAGreedy::ID = 0;
-char &llvm::RAGreedyID = RAGreedy::ID;
+namespace {
+class RAGreedyLegacy : public MachineFunctionPass {
+ RegAllocFilterFunc F;
-INITIALIZE_PASS_BEGIN(RAGreedy, "greedy",
- "Greedy Register Allocator", false, false)
+public:
+ RAGreedyLegacy(const RegAllocFilterFunc F = nullptr);
+
+ static char ID;
+ /// Return the pass name.
+ StringRef getPassName() const override { return "Greedy Register Allocator"; }
+
+ /// RAGreedy analysis usage.
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+ /// Perform register allocation.
+ bool runOnMachineFunction(MachineFunction &mf) override;
+
+ MachineFunctionProperties getRequiredProperties() const override {
+ return MachineFunctionProperties().set(
+ MachineFunctionProperties::Property::NoPHIs);
+ }
+
+ MachineFunctionProperties getClearedProperties() const override {
+ return MachineFunctionProperties().set(
+ MachineFunctionProperties::Property::IsSSA);
+ }
+};
+
+} // end anonymous namespace
+
+RAGreedyLegacy::RAGreedyLegacy(const RegAllocFilterFunc F)
+ : MachineFunctionPass(ID), F(F) {
+ initializeRAGreedyLegacyPass(*PassRegistry::getPassRegistry());
+}
+
+RAGreedy::RAGreedy(const RegAllocFilterFunc F) : RegAllocBase(F) {}
+
+void RAGreedy::setAnalyses(RequiredAnalyses &Analyses) {
+ VRM = Analyses.VRM;
+ LIS = Analyses.LIS;
+ Matrix = Analyses.LRM;
+ Indexes = Analyses.Indexes;
+ MBFI = Analyses.MBFI;
+ DomTree = Analyses.DomTree;
+ Loops = Analyses.Loops;
+ ORE = Analyses.ORE;
+ Bundles = Analyses.Bundles;
+ SpillPlacer = Analyses.SpillPlacer;
+ DebugVars = Analyses.DebugVars;
+ LSS = Analyses.LSS;
+ EvictProvider = Analyses.EvictProvider;
+ PriorityProvider = Analyses.PriorityProvider;
+}
+
+PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ MFPropsModifier _(*this, MF);
+
+ RAGreedy Impl(Filter);
+ RAGreedy::RequiredAnalyses Analyses;
+
+ Analyses.VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);
+ Analyses.LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ Analyses.LRM = &MFAM.getResult<LiveRegMatrixAnalysis>(MF);
+ Analyses.LSS = &MFAM.getResult<LiveStacksAnalysis>(MF);
+ Analyses.Indexes = &MFAM.getResult<SlotIndexesAnalysis>(MF);
+ Analyses.MBFI = &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF);
+ Analyses.DomTree = &MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
+ Analyses.ORE = &MFAM.getResult<MachineOptimizationRemarkEmitterAnalysis>(MF);
+ Analyses.Loops = &MFAM.getResult<MachineLoopAnalysis>(MF);
+ Analyses.Bundles = &MFAM.getResult<EdgeBundlesAnalysis>(MF);
+ Analyses.SpillPlacer = &MFAM.getResult<SpillPlacementAnalysis>(MF);
+ Analyses.DebugVars = &MFAM.getResult<LiveDebugVariablesAnalysis>(MF);
+ Analyses.EvictProvider =
+ MFAM.getResult<RegAllocEvictionAdvisorAnalysis>(MF).Provider;
+ Analyses.PriorityProvider =
+ MFAM.getResult<RegAllocPriorityAdvisorAnalysis>(MF).Provider;
+
+ Impl.setAnalyses(Analyses);
+ bool Changed = Impl.run(MF);
+ if (!Changed)
+ return PreservedAnalyses::all();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ PA.preserve<MachineBlockFrequencyAnalysis>();
+ PA.preserve<LiveIntervalsAnalysis>();
+ PA.preserve<SlotIndexesAnalysis>();
+ PA.preserve<LiveDebugVariablesAnalysis>();
+ PA.preserve<LiveStacksAnalysis>();
+ PA.preserve<MachineDominatorTreeAnalysis>();
+ PA.preserve<MachineLoopAnalysis>();
+ PA.preserve<VirtRegMapAnalysis>();
+ PA.preserve<LiveRegMatrixAnalysis>();
+ return PA;
+}
+
+bool RAGreedyLegacy::runOnMachineFunction(MachineFunction &MF) {
+ RAGreedy Impl(F);
+
+ RAGreedy::RequiredAnalyses Analyses;
+ Analyses.VRM = &getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
+ Analyses.LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+ Analyses.LSS = &getAnalysis<LiveStacksWrapperLegacy>().getLS();
+ Analyses.LRM = &getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
+ Analyses.Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
+ Analyses.MBFI =
+ &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
+ Analyses.DomTree =
+ &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ Analyses.ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
+ Analyses.Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
+ Analyses.Bundles = &getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
+ Analyses.SpillPlacer =
+ &getAnalysis<SpillPlacementWrapperLegacy>().getResult();
+ Analyses.DebugVars = &getAnalysis<LiveDebugVariablesWrapperLegacy>().getLDV();
+ Analyses.EvictProvider =
+ getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider().get();
+ Analyses.PriorityProvider =
+ &getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().getProvider();
+
+ Impl.setAnalyses(Analyses);
+ return Impl.run(MF);
+}
+
+char RAGreedyLegacy::ID = 0;
+char &llvm::RAGreedyLegacyID = RAGreedyLegacy::ID;
+
+INITIALIZE_PASS_BEGIN(RAGreedyLegacy, "greedy", "Greedy Register Allocator",
+ false, false)
INITIALIZE_PASS_DEPENDENCY(LiveDebugVariablesWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
@@ -166,8 +292,8 @@ INITIALIZE_PASS_DEPENDENCY(SpillPlacementWrapperLegacy)
INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysisLegacy)
INITIALIZE_PASS_DEPENDENCY(RegAllocPriorityAdvisorAnalysisLegacy)
-INITIALIZE_PASS_END(RAGreedy, "greedy",
- "Greedy Register Allocator", false, false)
+INITIALIZE_PASS_END(RAGreedyLegacy, "greedy", "Greedy Register Allocator",
+ false, false)
#ifndef NDEBUG
const char *const RAGreedy::StageName[] = {
@@ -186,17 +312,14 @@ const char *const RAGreedy::StageName[] = {
const float Hysteresis = (2007 / 2048.0f); // 0.97998046875
FunctionPass* llvm::createGreedyRegisterAllocator() {
- return new RAGreedy();
+ return new RAGreedyLegacy();
}
FunctionPass *llvm::createGreedyRegisterAllocator(RegAllocFilterFunc Ftor) {
- return new RAGreedy(Ftor);
+ return new RAGreedyLegacy(Ftor);
}
-RAGreedy::RAGreedy(RegAllocFilterFunc F)
- : MachineFunctionPass(ID), RegAllocBase(F) {}
-
-void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
+void RAGreedyLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
AU.addPreserved<MachineBlockFrequencyInfoWrapperPass>();
@@ -1057,7 +1180,8 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit,
}
if (VerifyEnabled)
- MF->verify(this, "After splitting live range around region", &errs());
+ MF->verify(LIS, Indexes, "After splitting live range around region",
+ &errs());
}
MCRegister RAGreedy::tryRegionSplit(const LiveInterval &VirtReg,
@@ -1326,7 +1450,8 @@ Register RAGreedy::tryBlockSplit(const LiveInterval &VirtReg,
}
if (VerifyEnabled)
- MF->verify(this, "After splitting live range around basic blocks", &errs());
+ MF->verify(LIS, Indexes, "After splitting live range around basic blocks",
+ &errs());
return Register();
}
@@ -2524,7 +2649,7 @@ MCRegister RAGreedy::selectOrSplitImpl(const LiveInterval &VirtReg,
DebugVars->splitRegister(r, LRE.regs(), *LIS);
if (VerifyEnabled)
- MF->verify(this, "After spilling", &errs());
+ MF->verify(LIS, Indexes, "After spilling", &errs());
}
// The live virtual register requesting allocation was spilled, so tell
@@ -2720,7 +2845,7 @@ bool RAGreedy::hasVirtRegAlloc() {
return false;
}
-bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
+bool RAGreedy::run(MachineFunction &mf) {
LLVM_DEBUG(dbgs() << "********** GREEDY REGISTER ALLOCATION **********\n"
<< "********** Function: " << mf.getName() << '\n');
@@ -2728,29 +2853,18 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
TII = MF->getSubtarget().getInstrInfo();
if (VerifyEnabled)
- MF->verify(this, "Before greedy register allocator", &errs());
+ MF->verify(LIS, Indexes, "Before greedy register allocator", &errs());
- RegAllocBase::init(getAnalysis<VirtRegMapWrapperLegacy>().getVRM(),
- getAnalysis<LiveIntervalsWrapperPass>().getLIS(),
- getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM());
+ RegAllocBase::init(*this->VRM, *this->LIS, *this->Matrix);
// Early return if there is no virtual register to be allocated to a
// physical register.
if (!hasVirtRegAlloc())
return false;
- Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
// Renumber to get accurate and consistent results from
// SlotIndexes::getApproxInstrDistance.
Indexes->packIndexes();
- MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
- DomTree = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
- ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
- Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
- Bundles = &getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
- SpillPlacer = &getAnalysis<SpillPlacementWrapperLegacy>().getResult();
- DebugVars = &getAnalysis<LiveDebugVariablesWrapperLegacy>().getLDV();
- auto &LSS = getAnalysis<LiveStacksWrapperLegacy>().getLS();
initializeCSRCost();
@@ -2766,17 +2880,12 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
ExtraInfo.emplace();
- auto &EvictAdvisorProvider =
- getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider();
- EvictAdvisor = EvictAdvisorProvider.getAdvisor(*MF, *this, MBFI, Loops);
-
- PriorityAdvisor = getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>()
- .getProvider()
- .getAdvisor(*MF, *this, *Indexes);
+ EvictAdvisor = EvictProvider->getAdvisor(*MF, *this, MBFI, Loops);
+ PriorityAdvisor = PriorityProvider->getAdvisor(*MF, *this, *Indexes);
VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI);
SpillerInstance.reset(
- createInlineSpiller({*LIS, LSS, *DomTree, *MBFI}, *MF, *VRM, *VRAI));
+ createInlineSpiller({*LIS, *LSS, *DomTree, *MBFI}, *MF, *VRM, *VRAI));
VRAI->calculateSpillWeightsAndHints();
@@ -2793,7 +2902,7 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
tryHintsRecoloring();
if (VerifyEnabled)
- MF->verify(this, "Before post optimization", &errs());
+ MF->verify(LIS, Indexes, "Before post optimization", &errs());
postOptimization();
reportStats();
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 1698607984bcd..7586d6abd18f5 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -25,13 +25,15 @@
#include "llvm/CodeGen/LiveDebugVariables.h"
#include "llvm/CodeGen/LiveInterval.h"
#include "llvm/CodeGen/LiveRangeEdit.h"
+#include "llvm/CodeGen/LiveStacks.h"
#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/SpillPlacement.h"
#include "llvm/CodeGen/Spiller.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/IR/PassManager.h"
#include <algorithm>
#include <cstdint>
#include <memory>
@@ -56,11 +58,30 @@ class SlotIndexes;
class TargetInstrInfo;
class VirtRegMap;
-class LLVM_LIBRARY_VISIBILITY RAGreedy : public MachineFunctionPass,
- public RegAllocBase,
+class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
private LiveRangeEdit::Delegate {
- // Interface to eviction advisers
public:
+ struct RequiredAnalyses {
+ VirtRegMap *VRM = nullptr;
+ LiveIntervals *LIS = nullptr;
+ LiveRegMatrix *LRM = nullptr;
+ SlotIndexes *Indexes = nullptr;
+ MachineBlockFrequencyInfo *MBFI = nullptr;
+ MachineDominatorTree *DomTree = nullptr;
+ MachineLoopInfo *Loops = nullptr;
+ MachineOptimizationRemarkEmitter *ORE = nullptr;
+ EdgeBundles *Bundles = nullptr;
+ SpillPlacement *SpillPlacer = nullptr;
+ LiveDebugVariables *DebugVars = nullptr;
+
+ // Used by InlineSpiller
+ LiveStacks *LSS;
+ // Proxies for eviction and priority advisors
+ RegAllocEvictionAdvisorProvider *EvictProvider;
+ RegAllocPriorityAdvisorProvider *PriorityProvider;
+ };
+
+ // Interface to eviction advisers
/// Track allocation stage and eviction loop prevention during allocation.
class ExtraRegInfo final {
// RegInfo - Keep additional information about each live range.
@@ -178,6 +199,10 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public MachineFunctionPass,
EdgeBundles *Bundles = nullptr;
SpillPlacement *SpillPlacer = nullptr;
LiveDebugVariables *DebugVars = nullptr;
+ LiveStacks *LSS = nullptr; // Used by InlineSpiller
+ // Proxy for the advisors
+ RegAllocEvictionAdvisorProvider *EvictProvider = nullptr;
+ RegAllocPriorityAdvisorProvider *PriorityProvider = nullptr;
// state
std::unique_ptr<Spiller> SpillerInstance;
@@ -282,13 +307,11 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public MachineFunctionPass,
public:
RAGreedy(const RegAllocFilterFunc F = nullptr);
+ // Evict and priority advisors use this object, so we can construct those
+ // first and pass them here.
+ // Not required once legacy PM is removed.
+ void setAnalyses(RequiredAnalyses &Analyses);
- /// Return the pass name.
- StringRef getPassName() const override { return "Greedy Register Allocator"; }
-
- /// RAGreedy analysis usage.
- void getAnalysisUsage(AnalysisUsage &AU) const override;
- void releaseMemory() override;
Spiller &spiller() override { return *SpillerInstance; }
void enqueueImpl(const LiveInterval *LI) override;
const LiveInterval *dequeue() override;
@@ -297,19 +320,9 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public MachineFunctionPass,
void aboutToRemoveInterval(const LiveInterval &) override;
/// Perform register allocation.
- bool runOnMachineFunction(MachineFunction &mf) override;
-
- MachineFunctionProperties getRequiredProperties() const override {
- return MachineFunctionProperties().set(
- MachineFunctionProperties::Property::NoPHIs);
- }
-
- MachineFunctionProperties getClearedProperties() const override {
- return MachineFunctionProperties().set(
- MachineFunctionProperties::Property::IsSSA);
- }
+ bool run(MachineFunction &mf);
- static char ID;
+ void releaseMemory();
private:
MCRegister selectOrSplitImpl(const LiveInterval &,
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 5bb2e7d0abdd9..455fac776597d 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -129,6 +129,7 @@
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/RegAllocFast.h"
+#include "llvm/CodeGen/RegAllocGreedyPass.h"
#include "llvm/CodeGen/RegAllocPriorityAdvisor.h"
#include "llvm/CodeGen/RegUsageInfoCollector.h"
#include "llvm/CodeGen/RegUsageInfoPropagate.h"
>From 6f2ba953ad1833b43196d66a23403ffe90a3bec2 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Fri, 13 Dec 2024 10:59:04 +0000
Subject: [PATCH 25/31] Move pass to a new header, add options
---
.../include/llvm/CodeGen/RegAllocGreedyPass.h | 43 +++++++++++++++++++
llvm/include/llvm/Passes/CodeGenPassBuilder.h | 1 +
.../llvm/Passes/MachinePassRegistry.def | 11 +++--
llvm/lib/CodeGen/RegAllocGreedy.cpp | 7 ++-
4 files changed, 55 insertions(+), 7 deletions(-)
create mode 100644 llvm/include/llvm/CodeGen/RegAllocGreedyPass.h
diff --git a/llvm/include/llvm/CodeGen/RegAllocGreedyPass.h b/llvm/include/llvm/CodeGen/RegAllocGreedyPass.h
new file mode 100644
index 0000000000000..f325224c5384c
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/RegAllocGreedyPass.h
@@ -0,0 +1,43 @@
+//==- RegAllocGreedyPass.h --- greedy register allocator pass ------*-C++-*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/RegAllocCommon.h"
+#include "llvm/CodeGen/RegAllocFast.h"
+#include "llvm/IR/PassManager.h"
+
+using namespace llvm;
+
+class RAGreedyPass : public PassInfoMixin<RAGreedyPass> {
+
+public:
+ struct Options {
+ RegAllocFilterFunc Filter;
+ StringRef FilterName;
+ Options(RegAllocFilterFunc F = nullptr, StringRef FN = "all")
+ : Filter(F), FilterName(FN) {};
+ };
+
+ RAGreedyPass(Options Opts = Options()) : Opts(Opts) {}
+ PreservedAnalyses run(MachineFunction &F, MachineFunctionAnalysisManager &AM);
+
+ MachineFunctionProperties getRequiredProperties() const {
+ return MachineFunctionProperties().set(
+ MachineFunctionProperties::Property::NoPHIs);
+ }
+
+ MachineFunctionProperties getClearedProperties() const {
+ return MachineFunctionProperties().set(
+ MachineFunctionProperties::Property::IsSSA);
+ }
+
+ void printPipeline(raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) const;
+ static bool isRequired() { return true; }
+
+private:
+ Options Opts;
+};
diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index 12781e2b84623..ca065d67eacef 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -59,6 +59,7 @@
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/CodeGen/RegAllocEvictionAdvisor.h"
#include "llvm/CodeGen/RegAllocFast.h"
+#include "llvm/CodeGen/RegAllocGreedyPass.h"
#include "llvm/CodeGen/RegUsageInfoCollector.h"
#include "llvm/CodeGen/RegUsageInfoPropagate.h"
#include "llvm/CodeGen/RegisterCoalescerPass.h"
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 78b4c8153e26b..9ac0f4f0b2555 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -196,12 +196,12 @@ MACHINE_FUNCTION_PASS_WITH_PARAMS(
"filter=reg-filter;no-clear-vregs")
MACHINE_FUNCTION_PASS_WITH_PARAMS(
- "regallocgreedy", "RAGreedy",
- [](RegAllocFilterFunc F) { return RAGreedyPass(F); },
+ "regallocgreedy", "RAGreedyPass",
+ [](RAGreedyPass::Options Opts) { return RAGreedyPass(Opts); },
[PB = this](StringRef Params) {
- // TODO: parseRegAllocFilter(*PB, Params);
- return Expected<RegAllocFilterFunc>(nullptr);
- }, ""
+ // TODO: parseRegAllocGreedyFilterFunc(*PB, Params);
+ return Expected<RAGreedyPass::Options>(RAGreedyPass::Options{});
+ }, "reg-filter"
)
#undef MACHINE_FUNCTION_PASS_WITH_PARAMS
@@ -269,7 +269,6 @@ DUMMY_MACHINE_FUNCTION_PASS("processimpdefs", ProcessImplicitDefsPass)
DUMMY_MACHINE_FUNCTION_PASS("prologepilog", PrologEpilogInserterPass)
DUMMY_MACHINE_FUNCTION_PASS("prologepilog-code", PrologEpilogCodeInserterPass)
DUMMY_MACHINE_FUNCTION_PASS("ra-basic", RABasicPass)
-DUMMY_MACHINE_FUNCTION_PASS("ra-greedy", RAGreedyPass)
DUMMY_MACHINE_FUNCTION_PASS("ra-pbqp", RAPBQPPass)
DUMMY_MACHINE_FUNCTION_PASS("regalloc", RegAllocPass)
DUMMY_MACHINE_FUNCTION_PASS("regallocscoringpass", RegAllocScoringPass)
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index f4cc80c751350..d193d2e7389df 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -202,11 +202,16 @@ void RAGreedy::setAnalyses(RequiredAnalyses &Analyses) {
PriorityProvider = Analyses.PriorityProvider;
}
+void RAGreedyPass::printPipeline(raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) const {
+ StringRef FilterName = Opts.FilterName.empty() ? "all" : Opts.FilterName;
+ OS << "regallocgreedy<" << FilterName << ">";
+}
+
PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
MFPropsModifier _(*this, MF);
- RAGreedy Impl(Filter);
+ RAGreedy Impl(Opts.Filter);
RAGreedy::RequiredAnalyses Analyses;
Analyses.VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);
>From f7855ebd07987bb2b42b77b5ed1d5cb707235c66 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Mon, 16 Dec 2024 09:34:30 +0000
Subject: [PATCH 26/31] Move VRM after LiveIntervals
---
llvm/lib/CodeGen/RegAllocGreedy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index d193d2e7389df..7eaf7254a0017 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -214,7 +214,6 @@ PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
RAGreedy Impl(Opts.Filter);
RAGreedy::RequiredAnalyses Analyses;
- Analyses.VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);
Analyses.LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
Analyses.LRM = &MFAM.getResult<LiveRegMatrixAnalysis>(MF);
Analyses.LSS = &MFAM.getResult<LiveStacksAnalysis>(MF);
@@ -230,6 +229,7 @@ PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
MFAM.getResult<RegAllocEvictionAdvisorAnalysis>(MF).Provider;
Analyses.PriorityProvider =
MFAM.getResult<RegAllocPriorityAdvisorAnalysis>(MF).Provider;
+ Analyses.VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);
Impl.setAnalyses(Analyses);
bool Changed = Impl.run(MF);
>From 01b21634c216d53536dbe309fb86124c516172a9 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Wed, 1 Jan 2025 08:51:35 +0000
Subject: [PATCH 27/31] AS: Rename to greedy, CFG obviation, comment fix
---
llvm/include/llvm/CodeGen/MachineFunction.h | 2 +-
llvm/include/llvm/Passes/MachinePassRegistry.def | 2 +-
llvm/lib/CodeGen/RegAllocGreedy.cpp | 4 +---
3 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 7fd0994883fe8..408428fe28a32 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -927,7 +927,7 @@ class LLVM_ABI MachineFunction {
/// Run the current MachineFunction through the machine code verifier, useful
/// for debugger use.
- /// TODO: Add the param LiveStks
+ /// TODO: Add the param for LiveStacks analysis.
/// \returns true if no problems were found.
bool verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
const char *Banner = nullptr, raw_ostream *OS = nullptr,
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 9ac0f4f0b2555..45abbc3b02e75 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -196,7 +196,7 @@ MACHINE_FUNCTION_PASS_WITH_PARAMS(
"filter=reg-filter;no-clear-vregs")
MACHINE_FUNCTION_PASS_WITH_PARAMS(
- "regallocgreedy", "RAGreedyPass",
+ "greedy", "RAGreedyPass",
[](RAGreedyPass::Options Opts) { return RAGreedyPass(Opts); },
[PB = this](StringRef Params) {
// TODO: parseRegAllocGreedyFilterFunc(*PB, Params);
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 7eaf7254a0017..717ae3afcc3db 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -242,8 +242,6 @@ PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
PA.preserve<SlotIndexesAnalysis>();
PA.preserve<LiveDebugVariablesAnalysis>();
PA.preserve<LiveStacksAnalysis>();
- PA.preserve<MachineDominatorTreeAnalysis>();
- PA.preserve<MachineLoopAnalysis>();
PA.preserve<VirtRegMapAnalysis>();
PA.preserve<LiveRegMatrixAnalysis>();
return PA;
@@ -269,7 +267,7 @@ bool RAGreedyLegacy::runOnMachineFunction(MachineFunction &MF) {
&getAnalysis<SpillPlacementWrapperLegacy>().getResult();
Analyses.DebugVars = &getAnalysis<LiveDebugVariablesWrapperLegacy>().getLDV();
Analyses.EvictProvider =
- getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider().get();
+ &getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider();
Analyses.PriorityProvider =
&getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().getProvider();
>From d5a9a5eee562fc56fbb3958a51ecf31ee951c2d9 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Thu, 23 Jan 2025 09:32:52 +0000
Subject: [PATCH 28/31] AS, remove setAnlayses and use constructor itself
---
llvm/include/llvm/CodeGen/RegAllocGreedyPass.h | 1 -
llvm/lib/CodeGen/RegAllocGreedy.cpp | 13 +++++++------
llvm/lib/CodeGen/RegAllocGreedy.h | 7 ++-----
3 files changed, 9 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/RegAllocGreedyPass.h b/llvm/include/llvm/CodeGen/RegAllocGreedyPass.h
index f325224c5384c..9fb6998921987 100644
--- a/llvm/include/llvm/CodeGen/RegAllocGreedyPass.h
+++ b/llvm/include/llvm/CodeGen/RegAllocGreedyPass.h
@@ -13,7 +13,6 @@
using namespace llvm;
class RAGreedyPass : public PassInfoMixin<RAGreedyPass> {
-
public:
struct Options {
RegAllocFilterFunc Filter;
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 717ae3afcc3db..69fa08cade063 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -183,7 +183,9 @@ RAGreedyLegacy::RAGreedyLegacy(const RegAllocFilterFunc F)
initializeRAGreedyLegacyPass(*PassRegistry::getPassRegistry());
}
-RAGreedy::RAGreedy(const RegAllocFilterFunc F) : RegAllocBase(F) {}
+RAGreedy::RAGreedy(RequiredAnalyses &Analyses, const RegAllocFilterFunc F) : RegAllocBase(F) {
+ setAnalyses(Analyses);
+}
void RAGreedy::setAnalyses(RequiredAnalyses &Analyses) {
VRM = Analyses.VRM;
@@ -204,14 +206,13 @@ void RAGreedy::setAnalyses(RequiredAnalyses &Analyses) {
void RAGreedyPass::printPipeline(raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) const {
StringRef FilterName = Opts.FilterName.empty() ? "all" : Opts.FilterName;
- OS << "regallocgreedy<" << FilterName << ">";
+ OS << "regallocgreedy<" << FilterName << '>';
}
PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
MFPropsModifier _(*this, MF);
- RAGreedy Impl(Opts.Filter);
RAGreedy::RequiredAnalyses Analyses;
Analyses.LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
@@ -231,7 +232,8 @@ PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
MFAM.getResult<RegAllocPriorityAdvisorAnalysis>(MF).Provider;
Analyses.VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);
- Impl.setAnalyses(Analyses);
+ RAGreedy Impl(Analyses, Opts.Filter);
+
bool Changed = Impl.run(MF);
if (!Changed)
return PreservedAnalyses::all();
@@ -248,7 +250,6 @@ PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
}
bool RAGreedyLegacy::runOnMachineFunction(MachineFunction &MF) {
- RAGreedy Impl(F);
RAGreedy::RequiredAnalyses Analyses;
Analyses.VRM = &getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
@@ -271,7 +272,7 @@ bool RAGreedyLegacy::runOnMachineFunction(MachineFunction &MF) {
Analyses.PriorityProvider =
&getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().getProvider();
- Impl.setAnalyses(Analyses);
+ RAGreedy Impl(Analyses, F);
return Impl.run(MF);
}
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 7586d6abd18f5..696a4009bec1b 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -305,12 +305,9 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
bool ReverseLocalAssignment = false;
-public:
- RAGreedy(const RegAllocFilterFunc F = nullptr);
- // Evict and priority advisors use this object, so we can construct those
- // first and pass them here.
- // Not required once legacy PM is removed.
void setAnalyses(RequiredAnalyses &Analyses);
+public:
+ RAGreedy(RequiredAnalyses &Analyses, const RegAllocFilterFunc F = nullptr);
Spiller &spiller() override { return *SpillerInstance; }
void enqueueImpl(const LiveInterval *LI) override;
>From e8a3bc9a4cb81fce3b2e6f5419adaf94653954f2 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Thu, 6 Feb 2025 04:17:13 +0000
Subject: [PATCH 29/31] remove setAnalyses()
---
llvm/lib/CodeGen/RegAllocGreedy.cpp | 9 +++------
llvm/lib/CodeGen/RegAllocGreedy.h | 1 -
2 files changed, 3 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 69fa08cade063..ebb85db210322 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -183,11 +183,8 @@ RAGreedyLegacy::RAGreedyLegacy(const RegAllocFilterFunc F)
initializeRAGreedyLegacyPass(*PassRegistry::getPassRegistry());
}
-RAGreedy::RAGreedy(RequiredAnalyses &Analyses, const RegAllocFilterFunc F) : RegAllocBase(F) {
- setAnalyses(Analyses);
-}
-
-void RAGreedy::setAnalyses(RequiredAnalyses &Analyses) {
+RAGreedy::RAGreedy(RequiredAnalyses &Analyses, const RegAllocFilterFunc F)
+ : RegAllocBase(F) {
VRM = Analyses.VRM;
LIS = Analyses.LIS;
Matrix = Analyses.LRM;
@@ -206,7 +203,7 @@ void RAGreedy::setAnalyses(RequiredAnalyses &Analyses) {
void RAGreedyPass::printPipeline(raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) const {
StringRef FilterName = Opts.FilterName.empty() ? "all" : Opts.FilterName;
- OS << "regallocgreedy<" << FilterName << '>';
+ OS << "regalloc-greedy<" << FilterName << '>';
}
PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index 696a4009bec1b..bb50f8a7f124e 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -305,7 +305,6 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
bool ReverseLocalAssignment = false;
- void setAnalyses(RequiredAnalyses &Analyses);
public:
RAGreedy(RequiredAnalyses &Analyses, const RegAllocFilterFunc F = nullptr);
>From d2885be6aeb13effbb512ccd125e4aa5fa14bcd5 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Wed, 12 Feb 2025 05:58:30 +0000
Subject: [PATCH 30/31] keep pass name same as legacy (greedy)
---
llvm/lib/CodeGen/RegAllocGreedy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index ebb85db210322..3dca5faacfe5d 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -203,7 +203,7 @@ RAGreedy::RAGreedy(RequiredAnalyses &Analyses, const RegAllocFilterFunc F)
void RAGreedyPass::printPipeline(raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) const {
StringRef FilterName = Opts.FilterName.empty() ? "all" : Opts.FilterName;
- OS << "regalloc-greedy<" << FilterName << '>';
+ OS << "greedy<" << FilterName << '>';
}
PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
>From ca1317d6c01b11a0a394894b4641749980c83fb6 Mon Sep 17 00:00:00 2001
From: Akshat Oke <Akshat.Oke at amd.com>
Date: Mon, 17 Feb 2025 08:42:14 +0000
Subject: [PATCH 31/31] Set analyses in constructors
---
llvm/lib/CodeGen/RegAllocGreedy.cpp | 81 ++++++++++++++---------------
llvm/lib/CodeGen/RegAllocGreedy.h | 4 ++
2 files changed, 44 insertions(+), 41 deletions(-)
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 3dca5faacfe5d..0fa67d423f96c 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -206,29 +206,30 @@ void RAGreedyPass::printPipeline(raw_ostream &OS, function_ref<StringRef(StringR
OS << "greedy<" << FilterName << '>';
}
+RAGreedy::RequiredAnalyses::RequiredAnalyses(
+ MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
+ LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ LRM = &MFAM.getResult<LiveRegMatrixAnalysis>(MF);
+ LSS = &MFAM.getResult<LiveStacksAnalysis>(MF);
+ Indexes = &MFAM.getResult<SlotIndexesAnalysis>(MF);
+ MBFI = &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF);
+ DomTree = &MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
+ ORE = &MFAM.getResult<MachineOptimizationRemarkEmitterAnalysis>(MF);
+ Loops = &MFAM.getResult<MachineLoopAnalysis>(MF);
+ Bundles = &MFAM.getResult<EdgeBundlesAnalysis>(MF);
+ SpillPlacer = &MFAM.getResult<SpillPlacementAnalysis>(MF);
+ DebugVars = &MFAM.getResult<LiveDebugVariablesAnalysis>(MF);
+ EvictProvider = MFAM.getResult<RegAllocEvictionAdvisorAnalysis>(MF).Provider;
+ PriorityProvider =
+ MFAM.getResult<RegAllocPriorityAdvisorAnalysis>(MF).Provider;
+ VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);
+}
+
PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
MFPropsModifier _(*this, MF);
- RAGreedy::RequiredAnalyses Analyses;
-
- Analyses.LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);
- Analyses.LRM = &MFAM.getResult<LiveRegMatrixAnalysis>(MF);
- Analyses.LSS = &MFAM.getResult<LiveStacksAnalysis>(MF);
- Analyses.Indexes = &MFAM.getResult<SlotIndexesAnalysis>(MF);
- Analyses.MBFI = &MFAM.getResult<MachineBlockFrequencyAnalysis>(MF);
- Analyses.DomTree = &MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
- Analyses.ORE = &MFAM.getResult<MachineOptimizationRemarkEmitterAnalysis>(MF);
- Analyses.Loops = &MFAM.getResult<MachineLoopAnalysis>(MF);
- Analyses.Bundles = &MFAM.getResult<EdgeBundlesAnalysis>(MF);
- Analyses.SpillPlacer = &MFAM.getResult<SpillPlacementAnalysis>(MF);
- Analyses.DebugVars = &MFAM.getResult<LiveDebugVariablesAnalysis>(MF);
- Analyses.EvictProvider =
- MFAM.getResult<RegAllocEvictionAdvisorAnalysis>(MF).Provider;
- Analyses.PriorityProvider =
- MFAM.getResult<RegAllocPriorityAdvisorAnalysis>(MF).Provider;
- Analyses.VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);
-
+ RAGreedy::RequiredAnalyses Analyses(MF, MFAM);
RAGreedy Impl(Analyses, Opts.Filter);
bool Changed = Impl.run(MF);
@@ -246,29 +247,27 @@ PreservedAnalyses RAGreedyPass::run(MachineFunction &MF,
return PA;
}
-bool RAGreedyLegacy::runOnMachineFunction(MachineFunction &MF) {
-
- RAGreedy::RequiredAnalyses Analyses;
- Analyses.VRM = &getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
- Analyses.LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
- Analyses.LSS = &getAnalysis<LiveStacksWrapperLegacy>().getLS();
- Analyses.LRM = &getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
- Analyses.Indexes = &getAnalysis<SlotIndexesWrapperPass>().getSI();
- Analyses.MBFI =
- &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
- Analyses.DomTree =
- &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
- Analyses.ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
- Analyses.Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
- Analyses.Bundles = &getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
- Analyses.SpillPlacer =
- &getAnalysis<SpillPlacementWrapperLegacy>().getResult();
- Analyses.DebugVars = &getAnalysis<LiveDebugVariablesWrapperLegacy>().getLDV();
- Analyses.EvictProvider =
- &getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider();
- Analyses.PriorityProvider =
- &getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().getProvider();
+RAGreedy::RequiredAnalyses::RequiredAnalyses(Pass &P) {
+ VRM = &P.getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
+ LIS = &P.getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+ LSS = &P.getAnalysis<LiveStacksWrapperLegacy>().getLS();
+ LRM = &P.getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
+ Indexes = &P.getAnalysis<SlotIndexesWrapperPass>().getSI();
+ MBFI = &P.getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();
+ DomTree = &P.getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ ORE = &P.getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
+ Loops = &P.getAnalysis<MachineLoopInfoWrapperPass>().getLI();
+ Bundles = &P.getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
+ SpillPlacer = &P.getAnalysis<SpillPlacementWrapperLegacy>().getResult();
+ DebugVars = &P.getAnalysis<LiveDebugVariablesWrapperLegacy>().getLDV();
+ EvictProvider =
+ &P.getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().getProvider();
+ PriorityProvider =
+ &P.getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().getProvider();
+}
+bool RAGreedyLegacy::runOnMachineFunction(MachineFunction &MF) {
+ RAGreedy::RequiredAnalyses Analyses(*this);
RAGreedy Impl(Analyses, F);
return Impl.run(MF);
}
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h
index bb50f8a7f124e..e9910c46a6452 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.h
+++ b/llvm/lib/CodeGen/RegAllocGreedy.h
@@ -79,6 +79,10 @@ class LLVM_LIBRARY_VISIBILITY RAGreedy : public RegAllocBase,
// Proxies for eviction and priority advisors
RegAllocEvictionAdvisorProvider *EvictProvider;
RegAllocPriorityAdvisorProvider *PriorityProvider;
+
+ RequiredAnalyses() {}
+ RequiredAnalyses(Pass &P);
+ RequiredAnalyses(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM);
};
// Interface to eviction advisers
More information about the llvm-branch-commits
mailing list