[llvm] Rebased save csr in ra (PR #131845)
Mikhail Gudim via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 18 10:58:58 PDT 2025
https://github.com/mgudim updated https://github.com/llvm/llvm-project/pull/131845
>From 28731f511eac1db0e469260df2b0b5f14b8fea6d Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Wed, 30 Oct 2024 10:49:13 -0700
Subject: [PATCH 1/2] WIP
>From ef9ee46cb01d0cb305a07704c0c6979805b2ccf6 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Fri, 1 Nov 2024 01:13:59 -0700
Subject: [PATCH 2/2] [RISCV][WIP] Let RA do the CSR saves.
We turn the problem of saving and restoring callee-saved registers efficiently into a register allocation problem. This has the advantage that the register allocator can essentialy do shrink-wrapping on per register basis. Currently, shrink-wrapping pass saves all CSR in the same place which may be suboptimal. Also, improvements to register allocation / coalescing will translate to improvements in shrink-wrapping.
In finalizeLowering() we copy all callee-saved registers from a physical register to a virtual one. In all return blocks we copy do the reverse.
---
.../llvm/CodeGen/ReachingDefAnalysis.h | 6 +
.../llvm/CodeGen/TargetFrameLowering.h | 24 +-
.../llvm/CodeGen/TargetSubtargetInfo.h | 2 +
.../llvm/DebugInfo/DWARF/DWARFDebugFrame.h | 10 +-
llvm/include/llvm/MC/MCDwarf.h | 37 +-
.../CodeGen/AsmPrinter/AsmPrinterDwarf.cpp | 4 +
llvm/lib/CodeGen/CFIInstrInserter.cpp | 359 +++++++++++-------
llvm/lib/CodeGen/CMakeLists.txt | 1 +
llvm/lib/CodeGen/MachineLICM.cpp | 48 ++-
llvm/lib/CodeGen/PrologEpilogInserter.cpp | 7 +
llvm/lib/CodeGen/ReachingDefAnalysis.cpp | 59 ++-
llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp | 19 +-
llvm/lib/CodeGen/TargetSubtargetInfo.cpp | 2 +
llvm/lib/MC/MCDwarf.cpp | 20 +
llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 220 ++++++++++-
llvm/lib/Target/RISCV/RISCVFrameLowering.h | 8 +
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 102 +++++
llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +
llvm/lib/Target/RISCV/RISCVInstrInfo.h | 7 +
llvm/lib/Target/RISCV/RISCVInstrInfo.td | 4 +
.../Target/RISCV/RISCVMachineFunctionInfo.h | 2 +
llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 8 +
llvm/lib/Target/RISCV/RISCVSubtarget.cpp | 7 +
llvm/lib/Target/RISCV/RISCVSubtarget.h | 2 +
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 2 +
25 files changed, 775 insertions(+), 187 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
index 0c1e707e4ecbb..e81f1dfa06dc7 100644
--- a/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
+++ b/llvm/include/llvm/CodeGen/ReachingDefAnalysis.h
@@ -114,8 +114,11 @@ class ReachingDefAnalysis : public MachineFunctionPass {
private:
MachineFunction *MF = nullptr;
const TargetRegisterInfo *TRI = nullptr;
+ const TargetInstrInfo *TII = nullptr;
LoopTraversal::TraversalOrder TraversedMBBOrder;
unsigned NumRegUnits = 0;
+ unsigned NumStackObjects = 0;
+ int ObjectIndexBegin = 0;
/// Instruction that defined each register, relative to the beginning of the
/// current basic block. When a LiveRegsDefInfo is used to represent a
/// live-out register, this value is relative to the end of the basic block,
@@ -138,6 +141,9 @@ class ReachingDefAnalysis : public MachineFunctionPass {
DenseMap<MachineInstr *, int> InstIds;
MBBReachingDefsInfo MBBReachingDefs;
+ using MBBFrameObjsReachingDefsInfo =
+ std::vector<std::vector<std::vector<int>>>;
+ MBBFrameObjsReachingDefsInfo MBBFrameObjsReachingDefs;
/// Default values are 'nothing happened a long time ago'.
const int ReachingDefDefaultVal = -(1 << 21);
diff --git a/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
index 97de0197da9b4..3d0a5151cb92e 100644
--- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
@@ -24,15 +24,16 @@ namespace llvm {
class CalleeSavedInfo;
class MachineFunction;
class RegScavenger;
-
-namespace TargetStackID {
-enum Value {
- Default = 0,
- SGPRSpill = 1,
- ScalableVector = 2,
- WasmLocal = 3,
- NoAlloc = 255
-};
+ class ReachingDefAnalysis;
+
+ namespace TargetStackID {
+ enum Value {
+ Default = 0,
+ SGPRSpill = 1,
+ ScalableVector = 2,
+ WasmLocal = 3,
+ NoAlloc = 255
+ };
}
/// Information about stack frame layout on the target. It holds the direction
@@ -210,6 +211,11 @@ class TargetFrameLowering {
/// for noreturn nounwind functions.
virtual bool enableCalleeSaveSkip(const MachineFunction &MF) const;
+ virtual void emitCFIsForCSRsHandledByRA(MachineFunction &MF,
+ ReachingDefAnalysis *RDA) const {
+ return;
+ }
+
/// emitProlog/emitEpilog - These methods insert prolog and epilog code into
/// the function.
virtual void emitPrologue(MachineFunction &MF,
diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
index a94ebf55f6c1e..ae0a14b89ea42 100644
--- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -328,6 +328,8 @@ class TargetSubtargetInfo : public MCSubtargetInfo {
return false;
}
+ virtual bool doCSRSavesInRA() const;
+
/// Classify a global function reference. This mainly used to fetch target
/// special flags for lowering a function address. For example mark a function
/// call should be plt or pc-related addressing.
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
index a9a3c7edde691..acf94cdb9ce50 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
@@ -74,8 +74,7 @@ class UnwindLocation {
bool Dereference; /// If true, the resulting location must be dereferenced
/// after the location value is computed.
- // Constructors are private to force people to use the create static
- // functions.
+public:
UnwindLocation(Location K)
: Kind(K), RegNum(InvalidRegisterNumber), Offset(0),
AddrSpace(std::nullopt), Dereference(false) {}
@@ -88,7 +87,6 @@ class UnwindLocation {
: Kind(DWARFExpr), RegNum(InvalidRegisterNumber), Offset(0), Expr(E),
Dereference(Deref) {}
-public:
/// Create a location whose rule is set to Unspecified. This means the
/// register value might be in the same register but it wasn't specified in
/// the unwind opcodes.
@@ -135,6 +133,7 @@ class UnwindLocation {
assert(Kind == RegPlusOffset && AddrSpace);
return *AddrSpace;
}
+ bool getDeref() const { return Dereference; }
int32_t getConstant() const { return Offset; }
/// Some opcodes will modify the CFA location's register only, so we need
/// to be able to modify the CFA register when evaluating DWARF Call Frame
@@ -148,6 +147,11 @@ class UnwindLocation {
/// the constant value (DW_CFA_GNU_window_save which is also known as
// DW_CFA_AARCH64_negate_ra_state).
void setConstant(int32_t Value) { Offset = Value; }
+ void setDeref(bool NewDeref) { Dereference = NewDeref; }
+ void setKind(Location NewKind) { Kind = NewKind; }
+ bool isRegister() const {
+ return ((Kind == RegPlusOffset) && !Dereference && (Offset == 0));
+ }
std::optional<DWARFExpression> getDWARFExpressionBytes() const {
return Expr;
diff --git a/llvm/include/llvm/MC/MCDwarf.h b/llvm/include/llvm/MC/MCDwarf.h
index 2fa7d73e1fa25..4e26d0488cd50 100644
--- a/llvm/include/llvm/MC/MCDwarf.h
+++ b/llvm/include/llvm/MC/MCDwarf.h
@@ -16,6 +16,7 @@
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/MC/StringTableBuilder.h"
@@ -504,6 +505,7 @@ class MCCFIInstruction {
OpRestoreState,
OpOffset,
OpLLVMDefAspaceCfa,
+ OpLLVMRegOffset,
OpDefCfaRegister,
OpDefCfaOffset,
OpDefCfa,
@@ -518,7 +520,7 @@ class MCCFIInstruction {
OpNegateRAStateWithPC,
OpGnuArgsSize,
OpLabel,
- OpValOffset,
+ OpValOffset
};
private:
@@ -537,6 +539,11 @@ class MCCFIInstruction {
unsigned Register;
unsigned Register2;
} RR;
+ struct {
+ unsigned Register;
+ unsigned Register2;
+ int64_t Offset;
+ } RRO;
MCSymbol *CfiLabel;
} U;
OpType Operation;
@@ -569,6 +576,13 @@ class MCCFIInstruction {
U.CfiLabel = CfiLabel;
}
+ MCCFIInstruction(OpType Op, MCSymbol *L, unsigned R, unsigned R2, int64_t O,
+ SMLoc Loc, StringRef V, StringRef Comment = "")
+ : Label(L), Operation(Op), Loc(Loc), Values(V.begin(), V.end()), Comment(Comment) {
+ assert(Op == OpLLVMRegOffset);
+ U.RRO = {R, R2, O};
+ }
+
public:
/// .cfi_def_cfa defines a rule for computing CFA as: take address from
/// Register and add Offset to it.
@@ -707,6 +721,15 @@ class MCCFIInstruction {
return MCCFIInstruction(OpValOffset, L, Register, Offset, Loc);
}
+ static void createRegOffsetExpression(unsigned Reg, unsigned FrameReg, int64_t Offset, SmallString<64>& CFAExpr);
+ static MCCFIInstruction createLLVMRegOffset(MCSymbol *L, unsigned Reg, unsigned FrameReg,
+ int64_t Offset, SMLoc Loc = {}, StringRef Comment = "") {
+ // Build up the expression (FrameRegister + Offset)
+ SmallString<64> CFAExpr;
+ createRegOffsetExpression(Reg, FrameReg, Offset, CFAExpr);
+ return MCCFIInstruction(OpLLVMRegOffset, L, Reg, FrameReg, Offset, Loc, CFAExpr, Comment);
+ }
+
OpType getOperation() const { return Operation; }
MCSymbol *getLabel() const { return Label; }
@@ -715,6 +738,8 @@ class MCCFIInstruction {
return U.RR.Register;
if (Operation == OpLLVMDefAspaceCfa)
return U.RIA.Register;
+ if (Operation == OpLLVMRegOffset)
+ return U.RRO.Register;
assert(Operation == OpDefCfa || Operation == OpOffset ||
Operation == OpRestore || Operation == OpUndefined ||
Operation == OpSameValue || Operation == OpDefCfaRegister ||
@@ -723,8 +748,10 @@ class MCCFIInstruction {
}
unsigned getRegister2() const {
- assert(Operation == OpRegister);
- return U.RR.Register2;
+ if (Operation == OpRegister)
+ return U.RR.Register2;
+ assert (Operation == OpLLVMRegOffset);
+ return U.RRO.Register2;
}
unsigned getAddressSpace() const {
@@ -735,6 +762,8 @@ class MCCFIInstruction {
int64_t getOffset() const {
if (Operation == OpLLVMDefAspaceCfa)
return U.RIA.Offset;
+ if (Operation == OpLLVMRegOffset)
+ return U.RRO.Offset;
assert(Operation == OpDefCfa || Operation == OpOffset ||
Operation == OpRelOffset || Operation == OpDefCfaOffset ||
Operation == OpAdjustCfaOffset || Operation == OpGnuArgsSize ||
@@ -748,7 +777,7 @@ class MCCFIInstruction {
}
StringRef getValues() const {
- assert(Operation == OpEscape);
+ assert(Operation == OpEscape || Operation == OpLLVMRegOffset);
return StringRef(&Values[0], Values.size());
}
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index 2a146eb15f709..46db960e7bd28 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -223,6 +223,10 @@ void AsmPrinter::emitCFIInstruction(const MCCFIInstruction &Inst) const {
OutStreamer->emitCFILLVMDefAspaceCfa(Inst.getRegister(), Inst.getOffset(),
Inst.getAddressSpace(), Loc);
break;
+ case MCCFIInstruction::OpLLVMRegOffset:
+ OutStreamer->AddComment(Inst.getComment());
+ OutStreamer->emitCFIEscape(Inst.getValues(), Loc);
+ break;
case MCCFIInstruction::OpOffset:
OutStreamer->emitCFIOffset(Inst.getRegister(), Inst.getOffset(), Loc);
break;
diff --git a/llvm/lib/CodeGen/CFIInstrInserter.cpp b/llvm/lib/CodeGen/CFIInstrInserter.cpp
index be8393cd38674..fc05e8955792a 100644
--- a/llvm/lib/CodeGen/CFIInstrInserter.cpp
+++ b/llvm/lib/CodeGen/CFIInstrInserter.cpp
@@ -24,6 +24,7 @@
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCDwarf.h"
@@ -55,11 +56,11 @@ class CFIInstrInserter : public MachineFunctionPass {
MBBVector.resize(MF.getNumBlockIDs());
calculateCFAInfo(MF);
- if (VerifyCFI) {
- if (unsigned ErrorNum = verify(MF))
- report_fatal_error("Found " + Twine(ErrorNum) +
- " in/out CFI information errors.");
- }
+ //if (VerifyCFI) {
+ // if (unsigned ErrorNum = verify(MF))
+ // report_fatal_error("Found " + Twine(ErrorNum) +
+ // " in/out CFI information errors.");
+ //}
bool insertedCFI = insertCFIInstrs(MF);
MBBVector.clear();
return insertedCFI;
@@ -76,10 +77,10 @@ class CFIInstrInserter : public MachineFunctionPass {
unsigned IncomingCFARegister = 0;
/// Value of cfa register valid at basic block exit.
unsigned OutgoingCFARegister = 0;
- /// Set of callee saved registers saved at basic block entry.
- BitVector IncomingCSRSaved;
- /// Set of callee saved registers saved at basic block exit.
- BitVector OutgoingCSRSaved;
+ /// Set of locations where the callee saved registers are at basic block entry.
+ SmallVector<dwarf::UnwindLocation> IncomingCSRLocations;
+ /// Set of locations where the callee saved registers are at basic block exit.
+ SmallVector<dwarf::UnwindLocation> OutgoingCSRLocations;
/// If in/out cfa offset and register values for this block have already
/// been set or not.
bool Processed = false;
@@ -162,10 +163,35 @@ void CFIInstrInserter::calculateCFAInfo(MachineFunction &MF) {
MBBInfo.OutgoingCFAOffset = InitialOffset;
MBBInfo.IncomingCFARegister = InitialRegister;
MBBInfo.OutgoingCFARegister = InitialRegister;
- MBBInfo.IncomingCSRSaved.resize(NumRegs);
- MBBInfo.OutgoingCSRSaved.resize(NumRegs);
+ MBBInfo.IncomingCSRLocations.resize(
+ NumRegs,
+ dwarf::UnwindLocation(
+ dwarf::UnwindLocation::RegPlusOffset,
+ (uint32_t) MCRegister::NoRegister,
+ (int32_t) 0,
+ std::nullopt,
+ false
+ )
+ );
+ MBBInfo.OutgoingCSRLocations.resize(
+ NumRegs,
+ dwarf::UnwindLocation(
+ dwarf::UnwindLocation::RegPlusOffset,
+ (uint32_t) 0,
+ (int32_t) 0,
+ std::nullopt,
+ false
+ )
+ );
+ }
+ MBBCFAInfo &EntryMBBInfo = MBBVector[MF.front().getNumber()];
+ const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
+ for (int i = 0; CSRegs[i]; ++i) {
+ unsigned Reg = TRI.getDwarfRegNum(CSRegs[i], true);
+ dwarf::UnwindLocation &CSRLoc = EntryMBBInfo.IncomingCSRLocations[Reg];
+ CSRLoc.setDeref(false);
+ CSRLoc.setRegister(Reg);
}
- CSRLocMap.clear();
// Set in/out cfa info for all blocks in the function. This traversal is based
// on the assumption that the first block in the function is the entry block
@@ -176,14 +202,17 @@ void CFIInstrInserter::calculateCFAInfo(MachineFunction &MF) {
void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) {
// Outgoing cfa offset set by the block.
- int64_t SetOffset = MBBInfo.IncomingCFAOffset;
+ int64_t &OutgoingCFAOffset = MBBInfo.OutgoingCFAOffset;
+ OutgoingCFAOffset = MBBInfo.IncomingCFAOffset;
// Outgoing cfa register set by the block.
- unsigned SetRegister = MBBInfo.IncomingCFARegister;
+ unsigned &OutgoingCFARegister = MBBInfo.OutgoingCFARegister;
+ OutgoingCFARegister = MBBInfo.IncomingCFARegister;
+ // Outgoing locations for each callee-saved register set by the block.
+ SmallVector<dwarf::UnwindLocation> &OutgoingCSRLocations = MBBInfo.OutgoingCSRLocations;
+ OutgoingCSRLocations = MBBInfo.IncomingCSRLocations;
+
MachineFunction *MF = MBBInfo.MBB->getParent();
const std::vector<MCCFIInstruction> &Instrs = MF->getFrameInstructions();
- const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
- unsigned NumRegs = TRI.getNumSupportedRegs(*MF);
- BitVector CSRSaved(NumRegs), CSRRestored(NumRegs);
#ifndef NDEBUG
int RememberState = 0;
@@ -192,38 +221,76 @@ void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) {
// Determine cfa offset and register set by the block.
for (MachineInstr &MI : *MBBInfo.MBB) {
if (MI.isCFIInstruction()) {
- std::optional<unsigned> CSRReg;
- std::optional<int64_t> CSROffset;
unsigned CFIIndex = MI.getOperand(0).getCFIIndex();
const MCCFIInstruction &CFI = Instrs[CFIIndex];
switch (CFI.getOperation()) {
- case MCCFIInstruction::OpDefCfaRegister:
- SetRegister = CFI.getRegister();
+ case MCCFIInstruction::OpDefCfaRegister: {
+ int Reg = CFI.getRegister();
+ assert(Reg >= 0 && "Negative dwarf register number!");
+ OutgoingCFARegister = Reg;
break;
- case MCCFIInstruction::OpDefCfaOffset:
- SetOffset = CFI.getOffset();
+ }
+ case MCCFIInstruction::OpDefCfaOffset: {
+ OutgoingCFAOffset = CFI.getOffset();
break;
- case MCCFIInstruction::OpAdjustCfaOffset:
- SetOffset += CFI.getOffset();
+ }
+ case MCCFIInstruction::OpAdjustCfaOffset: {
+ OutgoingCFAOffset += CFI.getOffset();
break;
- case MCCFIInstruction::OpDefCfa:
- SetRegister = CFI.getRegister();
- SetOffset = CFI.getOffset();
+ }
+ case MCCFIInstruction::OpDefCfa: {
+ int Reg = CFI.getRegister();
+ assert(Reg >= 0 && "Negative dwarf register number!");
+ OutgoingCFARegister = Reg;
+ OutgoingCFAOffset = CFI.getOffset();
break;
- case MCCFIInstruction::OpOffset:
- CSROffset = CFI.getOffset();
+ }
+ case MCCFIInstruction::OpOffset: {
+ int Reg = CFI.getRegister();
+ assert(Reg >= 0 && "Negative dwarf register number!");
+ dwarf::UnwindLocation &CSRLocation = OutgoingCSRLocations[Reg];
+ CSRLocation.setKind(dwarf::UnwindLocation::CFAPlusOffset);
+ CSRLocation.setOffset(CFI.getOffset());
+ CSRLocation.setDeref(true);
break;
- case MCCFIInstruction::OpRegister:
- CSRReg = CFI.getRegister2();
+ }
+ case MCCFIInstruction::OpRegister: {
+ int Reg = CFI.getRegister();
+ assert(Reg >= 0 && "Negative dwarf register number!");
+ int Reg2 = CFI.getRegister();
+ assert(Reg2 >= 0 && "Negative dwarf register number!");
+ dwarf::UnwindLocation &CSRLocation = OutgoingCSRLocations[Reg];
+ CSRLocation.setKind(dwarf::UnwindLocation::RegPlusOffset);
+ CSRLocation.setRegister((uint32_t) Reg2);
+ CSRLocation.setOffset(0);
+ CSRLocation.setDeref(false);
break;
- case MCCFIInstruction::OpRelOffset:
- CSROffset = CFI.getOffset() - SetOffset;
+ }
+ case MCCFIInstruction::OpLLVMRegOffset: {
+ int Reg = CFI.getRegister();
+ assert(Reg >= 0 && "Negative dwarf register number!");
+ int FrameReg = CFI.getRegister2();
+ assert(FrameReg >= 0 && "Negative dwarf register number!");
+ dwarf::UnwindLocation &CSRLocation = OutgoingCSRLocations[Reg];
+ CSRLocation.setKind(dwarf::UnwindLocation::RegPlusOffset);
+ CSRLocation.setRegister((uint32_t) FrameReg);
+ CSRLocation.setOffset(CFI.getOffset());
+ CSRLocation.setDeref(true);
break;
- case MCCFIInstruction::OpRestore:
- CSRRestored.set(CFI.getRegister());
+ }
+ case MCCFIInstruction::OpRestore: {
+ int Reg = CFI.getRegister();
+ assert(Reg >= 0 && "Negative dwarf register number!");
+ dwarf::UnwindLocation &CSRLocation = OutgoingCSRLocations[Reg];
+ CSRLocation.setKind(dwarf::UnwindLocation::RegPlusOffset);
+ CSRLocation.setRegister((uint32_t) Reg);
+ CSRLocation.setOffset(0);
+ CSRLocation.setDeref(false);
break;
+ }
+ // TODO: Add support for handling these:
+ case MCCFIInstruction::OpRelOffset:
case MCCFIInstruction::OpLLVMDefAspaceCfa:
- // TODO: Add support for handling cfi_def_aspace_cfa.
#ifndef NDEBUG
report_fatal_error(
"Support for cfi_llvm_def_aspace_cfa not implemented! Value of CFA "
@@ -266,16 +333,6 @@ void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) {
case MCCFIInstruction::OpValOffset:
break;
}
- if (CSRReg || CSROffset) {
- auto It = CSRLocMap.find(CFI.getRegister());
- if (It == CSRLocMap.end()) {
- CSRLocMap.insert(
- {CFI.getRegister(), CSRSavedLocation(CSRReg, CSROffset)});
- } else if (It->second.Reg != CSRReg || It->second.Offset != CSROffset) {
- llvm_unreachable("Different saved locations for the same CSR");
- }
- CSRSaved.set(CFI.getRegister());
- }
}
}
@@ -288,15 +345,6 @@ void CFIInstrInserter::calculateOutgoingCFAInfo(MBBCFAInfo &MBBInfo) {
#endif
MBBInfo.Processed = true;
-
- // Update outgoing CFA info.
- MBBInfo.OutgoingCFAOffset = SetOffset;
- MBBInfo.OutgoingCFARegister = SetRegister;
-
- // Update outgoing CSR info.
- BitVector::apply([](auto x, auto y, auto z) { return (x | y) & ~z; },
- MBBInfo.OutgoingCSRSaved, MBBInfo.IncomingCSRSaved, CSRSaved,
- CSRRestored);
}
void CFIInstrInserter::updateSuccCFAInfo(MBBCFAInfo &MBBInfo) {
@@ -312,7 +360,7 @@ void CFIInstrInserter::updateSuccCFAInfo(MBBCFAInfo &MBBInfo) {
if (!SuccInfo.Processed) {
SuccInfo.IncomingCFAOffset = CurrentInfo.OutgoingCFAOffset;
SuccInfo.IncomingCFARegister = CurrentInfo.OutgoingCFARegister;
- SuccInfo.IncomingCSRSaved = CurrentInfo.OutgoingCSRSaved;
+ SuccInfo.IncomingCSRLocations = CurrentInfo.OutgoingCSRLocations;
Stack.push_back(Succ);
}
}
@@ -320,11 +368,11 @@ void CFIInstrInserter::updateSuccCFAInfo(MBBCFAInfo &MBBInfo) {
}
bool CFIInstrInserter::insertCFIInstrs(MachineFunction &MF) {
- const MBBCFAInfo *PrevMBBInfo = &MBBVector[MF.front().getNumber()];
const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+ const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
+ const MBBCFAInfo *PrevMBBInfo = &MBBVector[MF.front().getNumber()];
bool InsertedCFIInstr = false;
- BitVector SetDifference;
for (MachineBasicBlock &MBB : MF) {
// Skip the first MBB in a function
if (MBB.getNumber() == MF.front().getNumber()) continue;
@@ -376,98 +424,121 @@ bool CFIInstrInserter::insertCFIInstrs(MachineFunction &MF) {
continue;
}
- BitVector::apply([](auto x, auto y) { return x & ~y; }, SetDifference,
- PrevMBBInfo->OutgoingCSRSaved, MBBInfo.IncomingCSRSaved);
- for (int Reg : SetDifference.set_bits()) {
- unsigned CFIIndex =
- MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, Reg));
- BuildMI(*MBBInfo.MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
- .addCFIIndex(CFIIndex);
- InsertedCFIInstr = true;
- }
-
- BitVector::apply([](auto x, auto y) { return x & ~y; }, SetDifference,
- MBBInfo.IncomingCSRSaved, PrevMBBInfo->OutgoingCSRSaved);
- for (int Reg : SetDifference.set_bits()) {
- auto it = CSRLocMap.find(Reg);
- assert(it != CSRLocMap.end() && "Reg should have an entry in CSRLocMap");
- unsigned CFIIndex;
- CSRSavedLocation RO = it->second;
- if (!RO.Reg && RO.Offset) {
- CFIIndex = MF.addFrameInst(
- MCCFIInstruction::createOffset(nullptr, Reg, *RO.Offset));
- } else if (RO.Reg && !RO.Offset) {
- CFIIndex = MF.addFrameInst(
- MCCFIInstruction::createRegister(nullptr, Reg, *RO.Reg));
- } else {
- llvm_unreachable("RO.Reg and RO.Offset cannot both be valid/invalid");
+ for (unsigned i = 0; i < PrevMBBInfo->OutgoingCSRLocations.size(); ++i) {
+ const dwarf::UnwindLocation &PrevOutgoingCSRLoc = PrevMBBInfo->OutgoingCSRLocations[i];
+ const dwarf::UnwindLocation &HasToBeCSRLoc = MBBInfo.IncomingCSRLocations[i];
+ // Ignore non-callee-saved registers, they remain uninitialized.
+ if (!HasToBeCSRLoc.getDeref() && (HasToBeCSRLoc.getRegister() == MCRegister::NoRegister))
+ continue;
+ if (HasToBeCSRLoc == PrevOutgoingCSRLoc)
+ continue;
+ unsigned CFIIndex = (unsigned)(-1);
+ if (HasToBeCSRLoc.getLocation() == dwarf::UnwindLocation::CFAPlusOffset) {
+ CFIIndex = MF.addFrameInst(
+ MCCFIInstruction::createOffset(nullptr, i, HasToBeCSRLoc.getOffset())
+ );
+ }
+ else if (
+ HasToBeCSRLoc.isRegister() &&
+ (HasToBeCSRLoc.getRegister() != MCRegister::NoRegister)
+ ) {
+ int NewReg = HasToBeCSRLoc.getRegister();
+ int DwarfEHReg = i;
+ if (NewReg == DwarfEHReg) {
+ CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
+ nullptr, DwarfEHReg));
+ }
+ else {
+ CFIIndex = MF.addFrameInst(
+ MCCFIInstruction::createRegister(nullptr, i, HasToBeCSRLoc.getRegister()));
+ }
+ }
+ else if(
+ (HasToBeCSRLoc.getLocation() == dwarf::UnwindLocation::RegPlusOffset) &&
+ HasToBeCSRLoc.getDeref()
+ ) {
+ int DwarfEHFrameReg = HasToBeCSRLoc.getRegister();
+ int DwarfEHReg = i;
+ int64_t FixedOffset = HasToBeCSRLoc.getOffset();
+
+ std::string CommentBuffer;
+ llvm::raw_string_ostream Comment(CommentBuffer);
+ Register LLVMReg = *TRI.getLLVMRegNum(DwarfEHReg, true);
+ Register LLVMFrameReg = *TRI.getLLVMRegNum(DwarfEHFrameReg, true);
+ Comment << printReg(LLVMReg, &TRI) << " = *(";
+ Comment << printReg(LLVMFrameReg, &TRI) << " + ";
+ Comment << FixedOffset << ")";
+ CFIIndex = MF.addFrameInst(MCCFIInstruction::createLLVMRegOffset(
+ nullptr, DwarfEHReg, DwarfEHFrameReg, FixedOffset, SMLoc(), Comment.str()));
+ }
+ else {
+ llvm_unreachable("Unexpected CSR location.");
}
BuildMI(*MBBInfo.MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
.addCFIIndex(CFIIndex);
InsertedCFIInstr = true;
}
-
PrevMBBInfo = &MBBInfo;
}
return InsertedCFIInstr;
}
-void CFIInstrInserter::reportCFAError(const MBBCFAInfo &Pred,
- const MBBCFAInfo &Succ) {
- errs() << "*** Inconsistent CFA register and/or offset between pred and succ "
- "***\n";
- errs() << "Pred: " << Pred.MBB->getName() << " #" << Pred.MBB->getNumber()
- << " in " << Pred.MBB->getParent()->getName()
- << " outgoing CFA Reg:" << Pred.OutgoingCFARegister << "\n";
- errs() << "Pred: " << Pred.MBB->getName() << " #" << Pred.MBB->getNumber()
- << " in " << Pred.MBB->getParent()->getName()
- << " outgoing CFA Offset:" << Pred.OutgoingCFAOffset << "\n";
- errs() << "Succ: " << Succ.MBB->getName() << " #" << Succ.MBB->getNumber()
- << " incoming CFA Reg:" << Succ.IncomingCFARegister << "\n";
- errs() << "Succ: " << Succ.MBB->getName() << " #" << Succ.MBB->getNumber()
- << " incoming CFA Offset:" << Succ.IncomingCFAOffset << "\n";
-}
-
-void CFIInstrInserter::reportCSRError(const MBBCFAInfo &Pred,
- const MBBCFAInfo &Succ) {
- errs() << "*** Inconsistent CSR Saved between pred and succ in function "
- << Pred.MBB->getParent()->getName() << " ***\n";
- errs() << "Pred: " << Pred.MBB->getName() << " #" << Pred.MBB->getNumber()
- << " outgoing CSR Saved: ";
- for (int Reg : Pred.OutgoingCSRSaved.set_bits())
- errs() << Reg << " ";
- errs() << "\n";
- errs() << "Succ: " << Succ.MBB->getName() << " #" << Succ.MBB->getNumber()
- << " incoming CSR Saved: ";
- for (int Reg : Succ.IncomingCSRSaved.set_bits())
- errs() << Reg << " ";
- errs() << "\n";
-}
-
-unsigned CFIInstrInserter::verify(MachineFunction &MF) {
- unsigned ErrorNum = 0;
- for (auto *CurrMBB : depth_first(&MF)) {
- const MBBCFAInfo &CurrMBBInfo = MBBVector[CurrMBB->getNumber()];
- for (MachineBasicBlock *Succ : CurrMBB->successors()) {
- const MBBCFAInfo &SuccMBBInfo = MBBVector[Succ->getNumber()];
- // Check that incoming offset and register values of successors match the
- // outgoing offset and register values of CurrMBB
- if (SuccMBBInfo.IncomingCFAOffset != CurrMBBInfo.OutgoingCFAOffset ||
- SuccMBBInfo.IncomingCFARegister != CurrMBBInfo.OutgoingCFARegister) {
- // Inconsistent offsets/registers are ok for 'noreturn' blocks because
- // we don't generate epilogues inside such blocks.
- if (SuccMBBInfo.MBB->succ_empty() && !SuccMBBInfo.MBB->isReturnBlock())
- continue;
- reportCFAError(CurrMBBInfo, SuccMBBInfo);
- ErrorNum++;
- }
- // Check that IncomingCSRSaved of every successor matches the
- // OutgoingCSRSaved of CurrMBB
- if (SuccMBBInfo.IncomingCSRSaved != CurrMBBInfo.OutgoingCSRSaved) {
- reportCSRError(CurrMBBInfo, SuccMBBInfo);
- ErrorNum++;
- }
- }
- }
- return ErrorNum;
-}
+//void CFIInstrInserter::reportCFAError(const MBBCFAInfo &Pred,
+// const MBBCFAInfo &Succ) {
+// errs() << "*** Inconsistent CFA register and/or offset between pred and succ "
+// "***\n";
+// errs() << "Pred: " << Pred.MBB->getName() << " #" << Pred.MBB->getNumber()
+// << " in " << Pred.MBB->getParent()->getName()
+// << " outgoing CFA Reg:" << Pred.OutgoingCFARegister << "\n";
+// errs() << "Pred: " << Pred.MBB->getName() << " #" << Pred.MBB->getNumber()
+// << " in " << Pred.MBB->getParent()->getName()
+// << " outgoing CFA Offset:" << Pred.OutgoingCFAOffset << "\n";
+// errs() << "Succ: " << Succ.MBB->getName() << " #" << Succ.MBB->getNumber()
+// << " incoming CFA Reg:" << Succ.IncomingCFARegister << "\n";
+// errs() << "Succ: " << Succ.MBB->getName() << " #" << Succ.MBB->getNumber()
+// << " incoming CFA Offset:" << Succ.IncomingCFAOffset << "\n";
+//}
+//
+//void CFIInstrInserter::reportCSRError(const MBBCFAInfo &Pred,
+// const MBBCFAInfo &Succ) {
+// errs() << "*** Inconsistent CSR Saved between pred and succ in function "
+// << Pred.MBB->getParent()->getName() << " ***\n";
+// errs() << "Pred: " << Pred.MBB->getName() << " #" << Pred.MBB->getNumber()
+// << " outgoing CSR Saved: ";
+// for (int Reg : Pred.OutgoingCSRSaved.set_bits())
+// errs() << Reg << " ";
+// errs() << "\n";
+// errs() << "Succ: " << Succ.MBB->getName() << " #" << Succ.MBB->getNumber()
+// << " incoming CSR Saved: ";
+// for (int Reg : Succ.IncomingCSRSaved.set_bits())
+// errs() << Reg << " ";
+// errs() << "\n";
+//}
+//
+//unsigned CFIInstrInserter::verify(MachineFunction &MF) {
+// unsigned ErrorNum = 0;
+// for (auto *CurrMBB : depth_first(&MF)) {
+// const MBBCFAInfo &CurrMBBInfo = MBBVector[CurrMBB->getNumber()];
+// for (MachineBasicBlock *Succ : CurrMBB->successors()) {
+// const MBBCFAInfo &SuccMBBInfo = MBBVector[Succ->getNumber()];
+// // Check that incoming offset and register values of successors match the
+// // outgoing offset and register values of CurrMBB
+// if (SuccMBBInfo.IncomingCFAOffset != CurrMBBInfo.OutgoingCFAOffset ||
+// SuccMBBInfo.IncomingCFARegister != CurrMBBInfo.OutgoingCFARegister) {
+// // Inconsistent offsets/registers are ok for 'noreturn' blocks because
+// // we don't generate epilogues inside such blocks.
+// if (SuccMBBInfo.MBB->succ_empty() && !SuccMBBInfo.MBB->isReturnBlock())
+// continue;
+// reportCFAError(CurrMBBInfo, SuccMBBInfo);
+// ErrorNum++;
+// }
+// // Check that IncomingCSRSaved of every successor matches the
+// // OutgoingCSRSaved of CurrMBB
+// if (SuccMBBInfo.IncomingCSRSaved != CurrMBBInfo.OutgoingCSRSaved) {
+// reportCSRError(CurrMBBInfo, SuccMBBInfo);
+// ErrorNum++;
+// }
+// }
+// }
+// return ErrorNum;
+//}
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 145fd2fac8b56..cd9e2bcd20be9 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -273,6 +273,7 @@ add_llvm_component_library(LLVMCodeGen
CGData
CodeGenTypes
Core
+ DebugInfoDWARF
MC
ObjCARC
ProfileData
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 1f6de0d6b2416..7c06fec8dce6b 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -262,15 +262,21 @@ namespace {
void HoistOutOfLoop(MachineDomTreeNode *HeaderN, MachineLoop *CurLoop,
MachineBasicBlock *CurPreheader);
- void InitRegPressure(MachineBasicBlock *BB);
+ void InitRegPressure(MachineBasicBlock *BB, const MachineLoop *Loop);
SmallDenseMap<unsigned, int> calcRegisterCost(const MachineInstr *MI,
bool ConsiderSeen,
- bool ConsiderUnseenAsDef);
+ bool ConsiderUnseenAsDef,
+ bool IgnoreDefs = false);
+ bool allDefsAreOnlyUsedOutsideOfTheLoop(const MachineInstr &MI,
+ const MachineLoop *Loop);
void UpdateRegPressure(const MachineInstr *MI,
- bool ConsiderUnseenAsDef = false);
+ bool ConsiderUnseenAsDef = false,
+ bool IgnoreDefs = false);
+ void UpdateRegPressureForUsesOnly(const MachineInstr *MI,
+ bool ConsiderUnseenAsDef = false);
MachineInstr *ExtractHoistableLoad(MachineInstr *MI, MachineLoop *CurLoop);
MachineInstr *LookForDuplicate(const MachineInstr *MI,
@@ -886,7 +892,7 @@ void MachineLICMImpl::HoistOutOfLoop(MachineDomTreeNode *HeaderN,
// Compute registers which are livein into the loop headers.
RegSeen.clear();
BackTrace.clear();
- InitRegPressure(Preheader);
+ InitRegPressure(Preheader, CurLoop);
// Now perform LICM.
for (MachineDomTreeNode *Node : Scopes) {
@@ -936,7 +942,8 @@ static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) {
/// Find all virtual register references that are liveout of the preheader to
/// initialize the starting "register pressure". Note this does not count live
/// through (livein but not used) registers.
-void MachineLICMImpl::InitRegPressure(MachineBasicBlock *BB) {
+void MachineLICMImpl::InitRegPressure(MachineBasicBlock *BB,
+ const MachineLoop *Loop) {
std::fill(RegPressure.begin(), RegPressure.end(), 0);
// If the preheader has only a single predecessor and it ends with a
@@ -947,17 +954,34 @@ void MachineLICMImpl::InitRegPressure(MachineBasicBlock *BB) {
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
SmallVector<MachineOperand, 4> Cond;
if (!TII->analyzeBranch(*BB, TBB, FBB, Cond, false) && Cond.empty())
- InitRegPressure(*BB->pred_begin());
+ InitRegPressure(*BB->pred_begin(), Loop);
}
- for (const MachineInstr &MI : *BB)
- UpdateRegPressure(&MI, /*ConsiderUnseenAsDef=*/true);
+ for (const MachineInstr &MI : *BB) {
+ bool IgnoreDefs = allDefsAreOnlyUsedOutsideOfTheLoop(MI, Loop);
+ UpdateRegPressure(&MI, /*ConsiderUnseenAsDef=*/true, IgnoreDefs);
+ }
+}
+
+bool MachineLICMImpl::allDefsAreOnlyUsedOutsideOfTheLoop(
+ const MachineInstr &MI, const MachineLoop *Loop) {
+ for (const MachineOperand DefMO : MI.all_defs()) {
+ if (!DefMO.isReg())
+ continue;
+ for (const MachineInstr &UseMI : MRI->use_instructions(DefMO.getReg())) {
+ if (Loop->contains(UseMI.getParent()))
+ return false;
+ }
+ }
+ return true;
}
/// Update estimate of register pressure after the specified instruction.
void MachineLICMImpl::UpdateRegPressure(const MachineInstr *MI,
- bool ConsiderUnseenAsDef) {
- auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/true, ConsiderUnseenAsDef);
+ bool ConsiderUnseenAsDef,
+ bool IgnoreDefs) {
+ auto Cost = calcRegisterCost(MI, /*ConsiderSeen=*/true, ConsiderUnseenAsDef,
+ IgnoreDefs);
for (const auto &RPIdAndCost : Cost) {
unsigned Class = RPIdAndCost.first;
if (static_cast<int>(RegPressure[Class]) < -RPIdAndCost.second)
@@ -975,7 +999,7 @@ void MachineLICMImpl::UpdateRegPressure(const MachineInstr *MI,
/// FIXME: Figure out a way to consider 'RegSeen' from all code paths.
SmallDenseMap<unsigned, int>
MachineLICMImpl::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen,
- bool ConsiderUnseenAsDef) {
+ bool ConsiderUnseenAsDef, bool IgnoreDefs) {
SmallDenseMap<unsigned, int> Cost;
if (MI->isImplicitDef())
return Cost;
@@ -993,7 +1017,7 @@ MachineLICMImpl::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen,
RegClassWeight W = TRI->getRegClassWeight(RC);
int RCCost = 0;
- if (MO.isDef())
+ if (MO.isDef() && !IgnoreDefs)
RCCost = W.RegWeight;
else {
bool isKill = isOperandKill(MO, MRI);
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index 34dd79c7b6184..58745e40f5746 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -36,6 +36,7 @@
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/ReachingDefAnalysis.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
@@ -92,6 +93,7 @@ class PEI : public MachineFunctionPass {
bool runOnMachineFunction(MachineFunction &MF) override;
private:
+ ReachingDefAnalysis *RDA = nullptr;
RegScavenger *RS = nullptr;
// MinCSFrameIndex, MaxCSFrameIndex - Keeps the range of callee saved
@@ -150,6 +152,7 @@ INITIALIZE_PASS_BEGIN(PEI, DEBUG_TYPE, "Prologue/Epilogue Insertion", false,
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
+INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis)
INITIALIZE_PASS_END(PEI, DEBUG_TYPE,
"Prologue/Epilogue Insertion & Frame Finalization", false,
false)
@@ -166,6 +169,7 @@ void PEI::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addPreserved<MachineLoopInfoWrapperPass>();
AU.addPreserved<MachineDominatorTreeWrapperPass>();
AU.addRequired<MachineOptimizationRemarkEmitterPass>();
+ AU.addRequired<ReachingDefAnalysis>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -224,6 +228,7 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) {
RS = TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr;
FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(MF);
ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
+ RDA = &getAnalysis<ReachingDefAnalysis>();
// Spill frame pointer and/or base pointer registers if they are clobbered.
// It is placed before call frame instruction elimination so it will not mess
@@ -259,6 +264,7 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) {
// called functions. Because of this, calculateCalleeSavedRegisters()
// must be called before this function in order to set the AdjustsStack
// and MaxCallFrameSize variables.
+ RDA->reset();
if (!F.hasFnAttribute(Attribute::Naked))
insertPrologEpilogCode(MF);
@@ -1161,6 +1167,7 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) {
void PEI::insertPrologEpilogCode(MachineFunction &MF) {
const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
+ TFI.emitCFIsForCSRsHandledByRA(MF, RDA);
// Add prologue to the function...
for (MachineBasicBlock *SaveBlock : SaveBlocks)
TFI.emitPrologue(MF, *SaveBlock);
diff --git a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
index 3ab6315f9c8ee..c64d1efb6f055 100644
--- a/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
+++ b/llvm/lib/CodeGen/ReachingDefAnalysis.cpp
@@ -10,6 +10,8 @@
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/CodeGen/LiveRegUnits.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/Support/Debug.h"
@@ -48,12 +50,28 @@ static bool isValidRegDefOf(const MachineOperand &MO, MCRegister Reg,
return TRI->regsOverlap(MO.getReg(), Reg);
}
+static bool isFIDef(const MachineInstr &MI, int FrameIndex,
+ const TargetInstrInfo *TII) {
+ int DefFrameIndex = 0;
+ int SrcFrameIndex = 0;
+ if (TII->isStoreToStackSlot(MI, DefFrameIndex) ||
+ TII->isStackSlotCopy(MI, DefFrameIndex, SrcFrameIndex)) {
+ return DefFrameIndex == FrameIndex;
+ }
+ return false;
+}
+
void ReachingDefAnalysis::enterBasicBlock(MachineBasicBlock *MBB) {
unsigned MBBNumber = MBB->getNumber();
assert(MBBNumber < MBBReachingDefs.numBlockIDs() &&
"Unexpected basic block number.");
MBBReachingDefs.startBasicBlock(MBBNumber, NumRegUnits);
+ MBBFrameObjsReachingDefs[MBBNumber].resize(NumStackObjects);
+ for (unsigned FOIdx = 0; FOIdx < NumStackObjects; ++FOIdx) {
+ MBBFrameObjsReachingDefs[MBBNumber][FOIdx].push_back(-1);
+ }
+
// Reset instruction counter in each basic block.
CurInstr = 0;
@@ -126,6 +144,13 @@ void ReachingDefAnalysis::processDefs(MachineInstr *MI) {
"Unexpected basic block number.");
for (auto &MO : MI->operands()) {
+ if (MO.isFI()) {
+ int FrameIndex = MO.getIndex();
+ if (!isFIDef(*MI, FrameIndex, TII))
+ continue;
+ MBBFrameObjsReachingDefs[MBBNumber][FrameIndex - ObjectIndexBegin]
+ .push_back(CurInstr);
+ }
if (!isValidRegDef(MO))
continue;
for (MCRegUnit Unit : TRI->regunits(MO.getReg().asMCReg())) {
@@ -211,7 +236,9 @@ void ReachingDefAnalysis::processBasicBlock(
bool ReachingDefAnalysis::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
- TRI = MF->getSubtarget().getRegisterInfo();
+ const TargetSubtargetInfo &STI = MF->getSubtarget();
+ TRI = STI.getRegisterInfo();
+ TII = STI.getInstrInfo();
LLVM_DEBUG(dbgs() << "********** REACHING DEFINITION ANALYSIS **********\n");
init();
traverse();
@@ -222,6 +249,7 @@ void ReachingDefAnalysis::releaseMemory() {
// Clear the internal vectors.
MBBOutRegsInfos.clear();
MBBReachingDefs.clear();
+ MBBFrameObjsReachingDefs.clear();
InstIds.clear();
LiveRegs.clear();
}
@@ -234,7 +262,10 @@ void ReachingDefAnalysis::reset() {
void ReachingDefAnalysis::init() {
NumRegUnits = TRI->getNumRegUnits();
+ NumStackObjects = MF->getFrameInfo().getNumObjects();
+ ObjectIndexBegin = MF->getFrameInfo().getObjectIndexBegin();
MBBReachingDefs.init(MF->getNumBlockIDs());
+ MBBFrameObjsReachingDefs.resize(MF->getNumBlockIDs());
// Initialize the MBBOutRegsInfos
MBBOutRegsInfos.resize(MF->getNumBlockIDs());
LoopTraversal Traversal;
@@ -269,6 +300,19 @@ int ReachingDefAnalysis::getReachingDef(MachineInstr *MI,
assert(MBBNumber < MBBReachingDefs.numBlockIDs() &&
"Unexpected basic block number.");
int LatestDef = ReachingDefDefaultVal;
+
+ if (Register::isStackSlot(Reg)) {
+ int FrameIndex = Register::stackSlot2Index(Reg);
+ for (int Def :
+ MBBFrameObjsReachingDefs[MBBNumber][FrameIndex - ObjectIndexBegin]) {
+ if (Def >= InstId)
+ break;
+ DefRes = Def;
+ }
+ LatestDef = std::max(LatestDef, DefRes);
+ return LatestDef;
+ }
+
for (MCRegUnit Unit : TRI->regunits(Reg)) {
for (int Def : MBBReachingDefs.defs(MBBNumber, Unit)) {
if (Def >= InstId)
@@ -422,7 +466,7 @@ void ReachingDefAnalysis::getLiveOuts(MachineBasicBlock *MBB, MCRegister Reg,
VisitedBBs.insert(MBB);
LiveRegUnits LiveRegs(*TRI);
LiveRegs.addLiveOuts(*MBB);
- if (LiveRegs.available(Reg))
+ if (Register::isPhysicalRegister(Reg) && LiveRegs.available(Reg))
return;
if (auto *Def = getLocalLiveOutMIDef(MBB, Reg))
@@ -505,7 +549,7 @@ bool ReachingDefAnalysis::isReachingDefLiveOut(MachineInstr *MI,
MachineBasicBlock *MBB = MI->getParent();
LiveRegUnits LiveRegs(*TRI);
LiveRegs.addLiveOuts(*MBB);
- if (LiveRegs.available(Reg))
+ if (Register::isPhysicalRegister(Reg) && LiveRegs.available(Reg))
return false;
auto Last = MBB->getLastNonDebugInstr();
@@ -525,7 +569,7 @@ MachineInstr *ReachingDefAnalysis::getLocalLiveOutMIDef(MachineBasicBlock *MBB,
MCRegister Reg) const {
LiveRegUnits LiveRegs(*TRI);
LiveRegs.addLiveOuts(*MBB);
- if (LiveRegs.available(Reg))
+ if (Register::isPhysicalRegister(Reg) && LiveRegs.available(Reg))
return nullptr;
auto Last = MBB->getLastNonDebugInstr();
@@ -533,6 +577,13 @@ MachineInstr *ReachingDefAnalysis::getLocalLiveOutMIDef(MachineBasicBlock *MBB,
return nullptr;
int Def = getReachingDef(&*Last, Reg);
+
+ if (Register::isStackSlot(Reg)) {
+ int FrameIndex = Register::stackSlot2Index(Reg);
+ if (isFIDef(*Last, FrameIndex, TII))
+ return &*Last;
+ }
+
for (auto &MO : Last->operands())
if (isValidRegDefOf(MO, Reg, TRI))
return &*Last;
diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
index a1f441ebd0d5e..44e545cf78015 100644
--- a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
+++ b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp
@@ -44,6 +44,13 @@ static cl::opt<bool> EnableLocalReassignment(
"may be compile time intensive"),
cl::init(false));
+static cl::opt<float> MinWeightRatioNeededToEvictHint(
+ "min-weight-ratio-needed-to-evict-hint", cl::Hidden,
+ cl::desc(
+ "The minimum ration of weight needed in order for a live range with "
+ "bigger weight to evict another live range which satisfies a hint"),
+ cl::init(1.0));
+
namespace llvm {
cl::opt<unsigned> EvictInterferenceCutoff(
"regalloc-eviction-max-interference-cutoff", cl::Hidden,
@@ -156,8 +163,16 @@ bool DefaultEvictionAdvisor::shouldEvict(const LiveInterval &A, bool IsHint,
if (CanSplit && IsHint && !BreaksHint)
return true;
- if (A.weight() > B.weight()) {
- LLVM_DEBUG(dbgs() << "should evict: " << B << '\n');
+ float AWeight = A.weight();
+ float BWeight = B.weight();
+ if (AWeight > BWeight) {
+ float WeightRatio = BWeight == 0.0 ? std::numeric_limits<float>::infinity()
+ : AWeight / BWeight;
+ if (CanSplit && !IsHint && BreaksHint &&
+ (WeightRatio < MinWeightRatioNeededToEvictHint)) {
+ return false;
+ }
+ LLVM_DEBUG(dbgs() << "should evict: " << B << " w= " << BWeight << '\n');
return true;
}
return false;
diff --git a/llvm/lib/CodeGen/TargetSubtargetInfo.cpp b/llvm/lib/CodeGen/TargetSubtargetInfo.cpp
index cd396e6a619a8..d7ff6abca5d36 100644
--- a/llvm/lib/CodeGen/TargetSubtargetInfo.cpp
+++ b/llvm/lib/CodeGen/TargetSubtargetInfo.cpp
@@ -46,6 +46,8 @@ bool TargetSubtargetInfo::enableRALocalReassignment(
return true;
}
+bool TargetSubtargetInfo::doCSRSavesInRA() const { return false; }
+
bool TargetSubtargetInfo::enablePostRAScheduler() const {
return getSchedModel().PostRAScheduler;
}
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index c17e9151ee487..adc99fd4b1689 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -41,6 +41,23 @@
using namespace llvm;
+void MCCFIInstruction::createRegOffsetExpression(unsigned Reg, unsigned FrameReg, int64_t Offset, SmallString<64>& CFAExpr) {
+ SmallString<64> Expr;
+ uint8_t Buffer[16];
+ Expr.push_back(dwarf::DW_OP_consts);
+ Expr.append(Buffer, Buffer + encodeSLEB128(Offset, Buffer));
+ Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
+ Expr.append(Buffer, Buffer + encodeULEB128(FrameReg, Buffer));
+ Expr.push_back(0);
+ Expr.push_back((uint8_t)dwarf::DW_OP_plus);
+ // Wrap this into DW_CFA_expression.
+ CFAExpr.push_back(dwarf::DW_CFA_expression);
+ CFAExpr.append(Buffer, Buffer + encodeULEB128(Reg, Buffer));
+ CFAExpr.append(Buffer, Buffer + encodeULEB128(Expr.size(), Buffer));
+ CFAExpr.append(Expr.str());
+ return;
+}
+
MCSymbol *mcdwarf::emitListsTableHeaderStart(MCStreamer &S) {
MCSymbol *Start = S.getContext().createTempSymbol("debug_list_header_start");
MCSymbol *End = S.getContext().createTempSymbol("debug_list_header_end");
@@ -1518,6 +1535,9 @@ void FrameEmitterImpl::emitCFIInstruction(const MCCFIInstruction &Instr) {
}
return;
}
+ case MCCFIInstruction::OpLLVMRegOffset:
+ Streamer.emitBytes(Instr.getValues());
+ return;
}
llvm_unreachable("Unhandled case in switch");
}
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 333c8060f37f4..f7d4ec796414d 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -19,12 +19,14 @@
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/ReachingDefAnalysis.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/Support/LEB128.h"
#include <algorithm>
+#include <unordered_set>
using namespace llvm;
@@ -786,6 +788,153 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
}
}
+struct CFIBuildInfo {
+ MachineBasicBlock *MBB;
+ MachineInstr *InsertAfterMI; // nullptr means insert at MBB.begin()
+ DebugLoc DL;
+ unsigned CFIIndex;
+};
+
+static void trackRegisterAndEmitCFIs(
+ MachineFunction &MF, MachineInstr &MI, MCRegister Reg, int DwarfEHRegNum,
+ const ReachingDefAnalysis &RDA, const TargetInstrInfo &TII,
+ const MachineFrameInfo &MFI, const RISCVRegisterInfo &TRI,
+ std::vector<CFIBuildInfo> &CFIBuildInfos,
+ std::unordered_set<MachineInstr *> &VisitedRestorePoints,
+ std::unordered_set<MachineInstr *> &VisitedDefs) {
+
+ if (VisitedRestorePoints.find(&MI) != VisitedRestorePoints.end()) {
+ return;
+ }
+ VisitedRestorePoints.insert(&MI);
+ SmallPtrSet<MachineInstr *, 2> Defs;
+ RDA.getGlobalReachingDefs(&MI, Reg, Defs);
+ MachineBasicBlock &EntryMBB = MF.front();
+ if (Defs.empty()) {
+ // it's a live-in register at the entry block.
+ // unsigned CFIIndex =
+ // MF.addFrameInst(MCCFIInstruction::createSameValue(nullptr,
+ // DwarfEHRegNum)); CFIBuildInfos.push_back({&EntryMBB, nullptr, DebugLoc(),
+ // CFIIndex});
+ return;
+ }
+
+ int FrameIndex = std::numeric_limits<int>::min();
+ for (MachineInstr *Def : Defs) {
+ if (VisitedDefs.find(Def) != VisitedDefs.end())
+ continue;
+ VisitedDefs.insert(Def);
+
+ MachineBasicBlock &MBB = *Def->getParent();
+ const DebugLoc &DL = Def->getDebugLoc();
+
+ if (Register StoredReg = TII.isStoreToStackSlot(*Def, FrameIndex)) {
+ assert(FrameIndex == Register::stackSlot2Index(Reg));
+
+ Register FrameReg;
+ StackOffset Offset =
+ MF.getSubtarget().getFrameLowering()->getFrameIndexReference(
+ MF, FrameIndex, FrameReg);
+ int64_t FixedOffset = Offset.getFixed();
+ // TODO:
+ assert(Offset.getScalable() == 0);
+
+ std::string CommentBuffer;
+ llvm::raw_string_ostream Comment(CommentBuffer);
+ int DwarfEHFrameReg = TRI.getDwarfRegNum(FrameReg, true);
+ Register LLVMReg = *TRI.getLLVMRegNum(DwarfEHRegNum, true);
+ Comment << printReg(LLVMReg, &TRI) << " = *(";
+ Comment << printReg(FrameReg, &TRI) << " + ";
+ Comment << FixedOffset << ")";
+ unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createLLVMRegOffset(
+ nullptr, DwarfEHRegNum, DwarfEHFrameReg, FixedOffset, SMLoc(), Comment.str()));
+
+ CFIBuildInfos.push_back({&MBB, Def, DL, CFIIndex});
+ trackRegisterAndEmitCFIs(MF, *Def, StoredReg, DwarfEHRegNum, RDA, TII,
+ MFI, TRI, CFIBuildInfos, VisitedRestorePoints,
+ VisitedDefs);
+ } else if (Register LoadedReg = TII.isLoadFromStackSlot(*Def, FrameIndex)) {
+ assert(LoadedReg == Reg);
+
+ unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRegister(
+ nullptr, DwarfEHRegNum, TRI.getDwarfRegNum(LoadedReg, true)));
+ CFIBuildInfos.push_back({&MBB, Def, DL, CFIIndex});
+ trackRegisterAndEmitCFIs(MF, *Def, Register::index2StackSlot(FrameIndex),
+ DwarfEHRegNum, RDA, TII, MFI, TRI, CFIBuildInfos,
+ VisitedRestorePoints, VisitedDefs);
+ } else if (auto DstSrc = TII.isCopyInstr(*Def)) {
+ Register DstReg = DstSrc->Destination->getReg();
+ Register SrcReg = DstSrc->Source->getReg();
+ assert(DstReg == Reg);
+
+ unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRegister(
+ nullptr, DwarfEHRegNum, TRI.getDwarfRegNum(DstReg, true)));
+ CFIBuildInfos.push_back({&MBB, Def, DL, CFIIndex});
+ trackRegisterAndEmitCFIs(MF, *Def, SrcReg, DwarfEHRegNum, RDA, TII, MFI,
+ TRI, CFIBuildInfos, VisitedRestorePoints,
+ VisitedDefs);
+ } else {
+ llvm_unreachable("Unexpected instruction");
+ }
+ }
+ return;
+}
+
+int RISCVFrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
+ return 0;
+}
+
+Register
+RISCVFrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
+ return RISCV::X2;
+}
+
+void RISCVFrameLowering::emitCFIsForCSRsHandledByRA(
+ MachineFunction &MF, ReachingDefAnalysis *RDA) const {
+ if (!STI.doCSRSavesInRA())
+ return;
+ const RISCVInstrInfo &TII = *STI.getInstrInfo();
+ const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
+ const MachineFrameInfo &MFI = MF.getFrameInfo();
+
+ BitVector MustCalleeSavedRegs;
+ determineMustCalleeSaves(MF, MustCalleeSavedRegs);
+ const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
+ SmallVector<MCPhysReg, 4> EligibleRegs;
+ for (int i = 0; CSRegs[i]; ++i) {
+ unsigned Reg = CSRegs[i];
+ if (!MustCalleeSavedRegs.test(Reg))
+ EligibleRegs.push_back(CSRegs[i]);
+ }
+
+ SmallVector<MachineInstr *, 4> RestorePoints;
+ for (MachineBasicBlock &MBB : MF) {
+ if (MBB.isReturnBlock())
+ RestorePoints.push_back(&MBB.back());
+ }
+ // TODO: replace CFIBuildInfo with UnwindLocation from DebugInfo/DWARF/DWARFDebugFrame
+ std::vector<CFIBuildInfo> CFIBuildInfos;
+ for (MCPhysReg Reg : EligibleRegs) {
+ std::unordered_set<MachineInstr *> VisitedDefs;
+ for (MachineInstr *RestorePoint : RestorePoints) {
+ std::unordered_set<MachineInstr *> VisitedRestorePoints;
+ trackRegisterAndEmitCFIs(
+ MF, *RestorePoint, Reg, TRI.getDwarfRegNum(Reg, true), *RDA, TII, MFI,
+ TRI, CFIBuildInfos, VisitedRestorePoints, VisitedDefs);
+ }
+ }
+ for (CFIBuildInfo &Info : CFIBuildInfos) {
+ MachineBasicBlock::iterator InsertPos =
+ Info.InsertAfterMI ? ++(Info.InsertAfterMI->getIterator())
+ : Info.MBB->begin();
+ BuildMI(*Info.MBB, InsertPos, Info.DL,
+ TII.get(TargetOpcode::CFI_INSTRUCTION))
+ .addCFIIndex(Info.CFIIndex)
+ .setMIFlag(MachineInstr::FrameSetup);
+ }
+ return;
+}
+
void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
MachineBasicBlock &MBB) const {
MachineFrameInfo &MFI = MF.getFrameInfo();
@@ -1398,17 +1547,55 @@ RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
return Offset;
}
-void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
- BitVector &SavedRegs,
- RegScavenger *RS) const {
- TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
- // Unconditionally spill RA and FP only if the function uses a frame
- // pointer.
+void RISCVFrameLowering::determineMustCalleeSaves(MachineFunction &MF,
+ BitVector &SavedRegs) const {
+ const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
+
+ // Resize before the early returns. Some backends expect that
+ // SavedRegs.size() == TRI.getNumRegs() after this call even if there are no
+ // saved registers.
+ SavedRegs.resize(TRI.getNumRegs());
+
+ // When interprocedural register allocation is enabled caller saved registers
+ // are preferred over callee saved registers.
+ if (MF.getTarget().Options.EnableIPRA &&
+ isSafeForNoCSROpt(MF.getFunction()) &&
+ isProfitableForNoCSROpt(MF.getFunction()))
+ return;
+
+ // Get the callee saved register list...
+ const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
+
+ // Early exit if there are no callee saved registers.
+ if (!CSRegs || CSRegs[0] == 0)
+ return;
+
+ // In Naked functions we aren't going to save any registers.
+ if (MF.getFunction().hasFnAttribute(Attribute::Naked))
+ return;
+
+ // Noreturn+nounwind functions never restore CSR, so no saves are needed.
+ // Purely noreturn functions may still return through throws, so those must
+ // save CSR for caller exception handlers.
+ //
+ // If the function uses longjmp to break out of its current path of
+ // execution we do not need the CSR spills either: setjmp stores all CSRs
+ // it was called with into the jmp_buf, which longjmp then restores.
+ if (MF.getFunction().hasFnAttribute(Attribute::NoReturn) &&
+ MF.getFunction().hasFnAttribute(Attribute::NoUnwind) &&
+ !MF.getFunction().hasFnAttribute(Attribute::UWTable) &&
+ enableCalleeSaveSkip(MF))
+ return;
+
+ // Functions which call __builtin_unwind_init get all their registers saved.
+ if (MF.callsUnwindInit()) {
+ SavedRegs.set();
+ return;
+ }
if (hasFP(MF)) {
- SavedRegs.set(RAReg);
- SavedRegs.set(FPReg);
+ SavedRegs.set(RISCV::X1);
+ SavedRegs.set(RISCV::X8);
}
- // Mark BP as used if function has dedicated base pointer.
if (hasBP(MF))
SavedRegs.set(RISCVABI::getBPReg());
@@ -1418,6 +1605,17 @@ void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
SavedRegs.set(RISCV::X27);
}
+void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
+ BitVector &SavedRegs,
+ RegScavenger *RS) const {
+ const auto &ST = MF.getSubtarget<RISCVSubtarget>();
+ determineMustCalleeSaves(MF, SavedRegs);
+ if (ST.doCSRSavesInRA())
+ return;
+
+ TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+}
+
std::pair<int64_t, Align>
RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
MachineFrameInfo &MFI = MF.getFrameInfo();
@@ -2122,6 +2320,10 @@ TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
return TargetStackID::ScalableVector;
}
+bool RISCVFrameLowering::enableCFIFixup(MachineFunction &MF) const {
+ return false;
+}
+
// Synthesize the probe loop.
static void emitStackProbeInline(MachineFunction &MF, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI, DebugLoc DL,
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.h b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
index d013755ce58a0..2f185051c8032 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
@@ -23,6 +23,12 @@ class RISCVFrameLowering : public TargetFrameLowering {
public:
explicit RISCVFrameLowering(const RISCVSubtarget &STI);
+ bool enableCFIFixup(MachineFunction &MF) const override;
+ int getInitialCFAOffset(const MachineFunction &MF) const override;
+ Register getInitialCFARegister(const MachineFunction &MF) const override;
+ void emitCFIsForCSRsHandledByRA(MachineFunction &MF,
+ ReachingDefAnalysis *RDA) const override;
+
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
@@ -31,6 +37,8 @@ class RISCVFrameLowering : public TargetFrameLowering {
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI,
Register &FrameReg) const override;
+ void determineMustCalleeSaves(MachineFunction &MF,
+ BitVector &SavedRegs) const;
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
RegScavenger *RS) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index de100c683a94f..55fbd5a702986 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -22601,6 +22601,108 @@ bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
return false;
}
+static MachineInstr *findInstrWhichNeedAllCSRs(MachineBasicBlock &MBB) {
+ // Some instructions may require (implicitly) all CSRs to be saved.
+ // For example, call to __cxa_throw is noreturn, but expects that all CSRs are
+ // taken care of.
+ // TODO: try to speedup this?
+ for (MachineInstr &MI : MBB) {
+ unsigned Opc = MI.getOpcode();
+ if (Opc != RISCV::PseudoCALL && Opc != RISCV::PseudoTAIL)
+ continue;
+ MachineOperand &MO = MI.getOperand(0);
+ StringRef Name = "";
+ if (MO.isSymbol()) {
+ Name = MO.getSymbolName();
+ } else if (MO.isGlobal()) {
+ Name = MO.getGlobal()->getName();
+ } else {
+ llvm_unreachable("Unexpected operand type.");
+ }
+ if (Name == "__cxa_throw" || Name == "__cxa_rethrow" ||
+ Name == "_Unwind_Resume")
+ return &MI;
+ }
+ return nullptr;
+}
+
+void RISCVTargetLowering::finalizeLowering(MachineFunction &MF) const {
+ if (!Subtarget.doCSRSavesInRA()) {
+ TargetLoweringBase::finalizeLowering(MF);
+ return;
+ }
+
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+ const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+ const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
+ const RISCVFrameLowering &TFI = *Subtarget.getFrameLowering();
+
+ SmallVector<MachineInstr *, 4> RestorePoints;
+ SmallVector<MachineBasicBlock *, 4> SaveMBBs;
+ SaveMBBs.push_back(&MF.front());
+ for (MachineBasicBlock &MBB : MF) {
+ if (MBB.isReturnBlock())
+ RestorePoints.push_back(&MBB.back());
+ if (MachineInstr *CallToCxaThrow = findInstrWhichNeedAllCSRs(MBB)) {
+ // MachineBasicBlock::iterator MII = CallToCxaThrow->getIterator();
+ //++MII;
+ // assert(MII->getOpcode() == RISCV::ADJCALLSTACKUP && "Unexpected
+ // instruction");
+ //++MII;
+ MachineBasicBlock::iterator MII = MBB.getFirstTerminator();
+ MachineInstr *NewRetMI = BuildMI(MBB, MII, CallToCxaThrow->getDebugLoc(),
+ TII.get(RISCV::UnreachableRET));
+ RestorePoints.push_back(NewRetMI);
+ MII = ++NewRetMI->getIterator();
+ MBB.erase(MII, MBB.end());
+ }
+ }
+
+ const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
+ SmallVector<MCPhysReg, 4> EligibleRegs;
+ BitVector MustCalleeSavedRegs;
+ TFI.determineMustCalleeSaves(MF, MustCalleeSavedRegs);
+ for (int i = 0; CSRegs[i]; ++i) {
+ unsigned Reg = CSRegs[i];
+ if (!MustCalleeSavedRegs.test(Reg)) {
+ EligibleRegs.push_back(CSRegs[i]);
+ }
+ }
+
+ SmallVector<Register, 4> VRegs;
+ for (MachineBasicBlock *SaveMBB : SaveMBBs) {
+ for (MCPhysReg Reg : EligibleRegs) {
+ SaveMBB->addLiveIn(Reg);
+ // TODO: should we use Maximal register class instead?
+ Register VReg = MRI.createVirtualRegister(
+ TRI.getLargestLegalSuperClass(TRI.getMinimalPhysRegClass(Reg), MF));
+ VRegs.push_back(VReg);
+ BuildMI(*SaveMBB, SaveMBB->begin(),
+ SaveMBB->findDebugLoc(SaveMBB->begin()),
+ TII.get(TargetOpcode::COPY), VReg)
+ .addReg(Reg);
+ MRI.setSimpleHint(VReg, Reg);
+ }
+ }
+
+ for (MachineInstr *RestorePoint : RestorePoints) {
+ auto VRegI = VRegs.begin();
+ for (MCPhysReg Reg : EligibleRegs) {
+ Register VReg = *VRegI;
+ BuildMI(*RestorePoint->getParent(), RestorePoint->getIterator(),
+ RestorePoint->getDebugLoc(), TII.get(TargetOpcode::COPY), Reg)
+ .addReg(VReg);
+ RestorePoint->addOperand(MF,
+ MachineOperand::CreateReg(Reg,
+ /*isDef=*/false,
+ /*isImplicit=*/true));
+ VRegI++;
+ }
+ }
+
+ TargetLoweringBase::finalizeLowering(MF);
+}
+
SDValue
RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
SelectionDAG &DAG,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 892c1cd96ca61..3b88cc76467fa 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -896,6 +896,8 @@ class RISCVTargetLowering : public TargetLowering {
bool fallBackToDAGISel(const Instruction &Inst) const override;
+ void finalizeLowering(MachineFunction &MF) const override;
+
bool lowerInterleavedLoad(LoadInst *LI,
ArrayRef<ShuffleVectorInst *> Shuffles,
ArrayRef<unsigned> Indices,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 7e8bcd451a8ef..8397c78a5ef3a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -300,6 +300,13 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+ bool expandPostRAPseudo(MachineInstr &MI) const override {
+ if (MI.getOpcode() == RISCV::UnreachableRET) {
+ MI.eraseFromParent();
+ return true;
+ }
+ return false;
+ }
protected:
const RISCVSubtarget &STI;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index bb5bb6352c32a..95c33aa2215f8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1680,6 +1680,10 @@ let isBarrier = 1, isReturn = 1, isTerminator = 1 in
def PseudoRET : Pseudo<(outs), (ins), [(riscv_ret_glue)]>,
PseudoInstExpansion<(JALR X0, X1, 0)>;
+let isBarrier = 1, isReturn = 1, isTerminator = 1, isMeta = 1, hasSideEffects = 1, mayLoad = 0, mayStore = 0 in
+def UnreachableRET : Pseudo<(outs), (ins), []>;
+
+
// PseudoTAIL is a pseudo instruction similar to PseudoCALL and will eventually
// expand to auipc and jalr while encoding.
// Define AsmString to print "tail" when compile with -S flag.
diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
index 27a13bb7cace1..e049fd784ffdf 100644
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -14,6 +14,7 @@
#define LLVM_LIB_TARGET_RISCV_RISCVMACHINEFUNCTIONINFO_H
#include "RISCVSubtarget.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/CodeGen/MIRYamlMapping.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -80,6 +81,7 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo {
/// Does it probe the stack for a dynamic allocation?
bool HasDynamicAllocation = false;
+ SmallDenseMap<MachineInstr *, std::tuple<int, int, int64_t>> CFIInfoMap;
public:
RISCVMachineFunctionInfo(const Function &F, const RISCVSubtarget *STI);
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index b9c70fe60fb50..945994af13edd 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -757,6 +757,14 @@ RISCVRegisterInfo::getCallPreservedMask(const MachineFunction & MF,
const TargetRegisterClass *
RISCVRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC,
const MachineFunction &) const {
+ if (RC == &RISCV::GPRX1RegClass)
+ return &RISCV::GPRRegClass;
+ if (RC == &RISCV::GPRCRegClass)
+ return &RISCV::GPRRegClass;
+ if (RC == &RISCV::SR07RegClass)
+ return &RISCV::GPRRegClass;
+ if (RC == &RISCV::GPRJALRRegClass)
+ return &RISCV::GPRRegClass;
if (RC == &RISCV::VMV0RegClass)
return &RISCV::VRRegClass;
if (RC == &RISCV::VRNoV0RegClass)
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
index 6e212dc58e6dd..33a7459d8bcf7 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -62,6 +62,11 @@ static cl::opt<unsigned> RISCVMinimumJumpTableEntries(
"riscv-min-jump-table-entries", cl::Hidden,
cl::desc("Set minimum number of entries to use a jump table on RISCV"));
+static cl::opt<bool> RISCVEnableSaveCSRByRA(
+ "riscv-enable-save-csr-in-ra",
+ cl::desc("Let register alloctor do csr saves/restores"), cl::init(false),
+ cl::Hidden);
+
void RISCVSubtarget::anchor() {}
RISCVSubtarget &
@@ -138,6 +143,8 @@ bool RISCVSubtarget::useConstantPoolForLargeInts() const {
return !RISCVDisableUsingConstantPoolForLargeInts;
}
+bool RISCVSubtarget::doCSRSavesInRA() const { return RISCVEnableSaveCSRByRA; }
+
unsigned RISCVSubtarget::getMaxBuildIntsCost() const {
// Loading integer from constant pool needs two instructions (the reason why
// the minimum cost is 2): an address calculation instruction and a load
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 87d508c394173..362c3911da64a 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -315,6 +315,8 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
bool useConstantPoolForLargeInts() const;
+ bool doCSRSavesInRA() const override;
+
// Maximum cost used for building integers, integers will be put into constant
// pool if exceeded.
unsigned getMaxBuildIntsCost() const;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index f6ccbfbe217df..1cecff80148f9 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -582,6 +582,8 @@ void RISCVPassConfig::addPreEmitPass2() {
addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
return MF.getFunction().getParent()->getModuleFlag("kcfi");
}));
+
+ addPass(createCFIInstrInserter());
}
void RISCVPassConfig::addMachineSSAOptimization() {
More information about the llvm-commits
mailing list