[llvm] 9beb467 - MC: Store fragment content and fixups out-of-line
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 1 00:21:17 PDT 2025
Author: Fangrui Song
Date: 2025-07-01T00:21:12-07:00
New Revision: 9beb467d9213fe2becdb0f1469d99ca058189ac7
URL: https://github.com/llvm/llvm-project/commit/9beb467d9213fe2becdb0f1469d99ca058189ac7
DIFF: https://github.com/llvm/llvm-project/commit/9beb467d9213fe2becdb0f1469d99ca058189ac7.diff
LOG: MC: Store fragment content and fixups out-of-line
Moved `Contents` and `Fixups` SmallVector storage to MCSection, enabling
trivial destructors for most fragment subclasses and eliminating the need
for MCFragment::destroy in ~MCSection.
For appending content to the current section, use
getContentsForAppending. During assembler relaxation, prefer
setContents/setFixups, which may involve copying and reduce the benefits
of https://reviews.llvm.org/D145791.
Moving only Contents out-of-line caused a slight performance regression
(Alexis Engelke's 2024 prototype). By also moving Fragments out-of-line,
fragment destructors become trivial, resulting in
neglgible instructions:u increase for "stage2-O0-g" and [large max-rss decrease](https://llvm-compile-time-tracker.com/compare.php?from=84e82746c3ff63ec23a8b85e9efd4f7fccf92590&to=555a28c0b2f8250a9cf86fd267a04b0460283e15&stat=max-rss&linkStats=on)
for the "stage1-ReleaseLTO-g (link only)" benchmark.
(
An older version using fewer inline functions: https://llvm-compile-time-tracker.com/compare.php?from=bb982e733cfcda7e4cfb0583544f68af65211ed1&to=f12d55f97c47717d438951ecddecf8ebd28c296b&linkStats=on
)
Now using plain SmallVector in MCSection for storage, with potential for
future allocator optimizations, such as allocating `Contents` as the
trailing object of MCDataFragment. (GNU Assembler uses gnulib's obstack
for fragment management.)
Co-authored-by: Alexis Engelke <engelke at in.tum.de>
Pull Request: https://github.com/llvm/llvm-project/pull/146307
Added:
Modified:
llvm/include/llvm/MC/MCSection.h
llvm/lib/MC/MCAssembler.cpp
llvm/lib/MC/MCCodeView.cpp
llvm/lib/MC/MCELFStreamer.cpp
llvm/lib/MC/MCFragment.cpp
llvm/lib/MC/MCObjectStreamer.cpp
llvm/lib/MC/MCSection.cpp
llvm/lib/MC/MachObjectWriter.cpp
llvm/lib/MC/WinCOFFObjectWriter.cpp
llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 7451ed6035471..3d430c0fc58e3 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -132,6 +132,7 @@ class LLVM_ABI MCSection {
public:
friend MCAssembler;
friend MCObjectStreamer;
+ friend class MCEncodedFragment;
static constexpr unsigned NonUniqueID = ~0U;
enum SectionVariant {
@@ -209,6 +210,10 @@ class LLVM_ABI MCSection {
// subsections.
SmallVector<std::pair<unsigned, FragList>, 1> Subsections;
+ // Content and fixup storage for fragments
+ SmallVector<char, 0> ContentStorage;
+ SmallVector<MCFixup, 0> FixupStorage;
+
protected:
// TODO Make Name private when possible.
StringRef Name;
@@ -296,9 +301,12 @@ class LLVM_ABI MCSection {
/// Interface implemented by fragments that contain encoded instructions and/or
/// data.
-///
class MCEncodedFragment : public MCFragment {
uint8_t BundlePadding = 0;
+ uint32_t ContentStart = 0;
+ uint32_t ContentEnd = 0;
+ uint32_t FixupStart = 0;
+ uint32_t FixupEnd = 0;
protected:
MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions)
@@ -318,7 +326,10 @@ class MCEncodedFragment : public MCFragment {
case MCFragment::FT_Data:
case MCFragment::FT_Dwarf:
case MCFragment::FT_DwarfFrame:
+ case MCFragment::FT_LEB:
case MCFragment::FT_PseudoProbe:
+ case MCFragment::FT_CVInlineLines:
+ case MCFragment::FT_CVDefRange:
return true;
}
}
@@ -348,48 +359,64 @@ class MCEncodedFragment : public MCFragment {
HasInstructions = true;
this->STI = &STI;
}
-};
-
-/// Interface implemented by fragments that contain encoded instructions and/or
-/// data and also have fixups registered.
-///
-template <unsigned ContentsSize, unsigned FixupsSize>
-class MCEncodedFragmentWithFixups : public MCEncodedFragment {
- SmallVector<char, ContentsSize> Contents;
-
- /// The list of fixups in this fragment.
- SmallVector<MCFixup, FixupsSize> Fixups;
-protected:
- MCEncodedFragmentWithFixups(MCFragment::FragmentType FType,
- bool HasInstructions)
- : MCEncodedFragment(FType, HasInstructions) {}
-
-public:
- SmallVectorImpl<char> &getContents() { return Contents; }
- const SmallVectorImpl<char> &getContents() const { return Contents; }
-
- void appendContents(ArrayRef<char> C) { Contents.append(C.begin(), C.end()); }
- void appendContents(size_t Num, char Elt) { Contents.append(Num, Elt); }
- void setContents(ArrayRef<char> C) { Contents.assign(C.begin(), C.end()); }
-
- void addFixup(MCFixup Fixup) { Fixups.push_back(Fixup); }
- SmallVectorImpl<MCFixup> &getFixups() { return Fixups; }
- const SmallVectorImpl<MCFixup> &getFixups() const { return Fixups; }
+ // Content-related functions manage parent's storage using ContentStart and
+ // ContentSize.
+ void clearContents() { ContentEnd = ContentStart; }
+ // Get a SmallVector reference. The caller should call doneAppending to update
+ // `ContentEnd`.
+ SmallVectorImpl<char> &getContentsForAppending() {
+ SmallVectorImpl<char> &S = getParent()->ContentStorage;
+ if (LLVM_UNLIKELY(ContentEnd != S.size())) {
+ // Move the elements to the end. Reserve space to avoid invalidating
+ // S.begin()+I for `append`.
+ auto Size = ContentEnd - ContentStart;
+ auto I = std::exchange(ContentStart, S.size());
+ S.reserve(S.size() + Size);
+ S.append(S.begin() + I, S.begin() + I + Size);
+ }
+ return S;
+ }
+ void doneAppending() { ContentEnd = getParent()->ContentStorage.size(); }
+ void appendContents(ArrayRef<char> Contents) {
+ getContentsForAppending().append(Contents.begin(), Contents.end());
+ doneAppending();
+ }
+ void appendContents(size_t Num, char Elt) {
+ getContentsForAppending().append(Num, Elt);
+ doneAppending();
+ }
+ void setContents(ArrayRef<char> Contents);
+ MutableArrayRef<char> getContents() {
+ return MutableArrayRef(getParent()->ContentStorage)
+ .slice(ContentStart, ContentEnd - ContentStart);
+ }
+ ArrayRef<char> getContents() const {
+ return ArrayRef(getParent()->ContentStorage)
+ .slice(ContentStart, ContentEnd - ContentStart);
+ }
- static bool classof(const MCFragment *F) {
- MCFragment::FragmentType Kind = F->getKind();
- return Kind == MCFragment::FT_Relaxable || Kind == MCFragment::FT_Data ||
- Kind == MCFragment::FT_CVDefRange || Kind == MCFragment::FT_Dwarf ||
- Kind == MCFragment::FT_DwarfFrame;
+ // Fixup-related functions manage parent's storage using FixupStart and
+ // FixupSize.
+ void clearFixups() { FixupEnd = FixupStart; }
+ void addFixup(MCFixup Fixup);
+ void appendFixups(ArrayRef<MCFixup> Fixups);
+ void setFixups(ArrayRef<MCFixup> Fixups);
+ MutableArrayRef<MCFixup> getFixups() {
+ return MutableArrayRef(getParent()->FixupStorage)
+ .slice(FixupStart, FixupEnd - FixupStart);
+ }
+ ArrayRef<MCFixup> getFixups() const {
+ return ArrayRef(getParent()->FixupStorage)
+ .slice(FixupStart, FixupEnd - FixupStart);
}
};
/// Fragment for data and encoded instructions.
///
-class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> {
+class MCDataFragment : public MCEncodedFragment {
public:
- MCDataFragment() : MCEncodedFragmentWithFixups<32, 4>(FT_Data, false) {}
+ MCDataFragment() : MCEncodedFragment(FT_Data, false) {}
static bool classof(const MCFragment *F) {
return F->getKind() == MCFragment::FT_Data;
@@ -402,13 +429,13 @@ class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> {
/// A relaxable fragment holds on to its MCInst, since it may need to be
/// relaxed during the assembler layout and relaxation stage.
///
-class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> {
+class MCRelaxableFragment : public MCEncodedFragment {
/// The instruction this is a fragment for.
MCInst Inst;
public:
MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI)
- : MCEncodedFragmentWithFixups(FT_Relaxable, true), Inst(Inst) {
+ : MCEncodedFragment(FT_Relaxable, true), Inst(Inst) {
this->STI = &STI;
}
@@ -557,7 +584,7 @@ class MCOrgFragment : public MCFragment {
}
};
-class MCLEBFragment final : public MCEncodedFragmentWithFixups<8, 0> {
+class MCLEBFragment final : public MCEncodedFragment {
/// True if this is a sleb128, false if uleb128.
bool IsSigned;
@@ -566,24 +593,19 @@ class MCLEBFragment final : public MCEncodedFragmentWithFixups<8, 0> {
public:
MCLEBFragment(const MCExpr &Value, bool IsSigned)
- : MCEncodedFragmentWithFixups<8, 0>(FT_LEB, false), IsSigned(IsSigned),
- Value(&Value) {
- getContents().push_back(0);
- }
+ : MCEncodedFragment(FT_LEB, false), IsSigned(IsSigned), Value(&Value) {}
const MCExpr &getValue() const { return *Value; }
void setValue(const MCExpr *Expr) { Value = Expr; }
bool isSigned() const { return IsSigned; }
- /// @}
-
static bool classof(const MCFragment *F) {
return F->getKind() == MCFragment::FT_LEB;
}
};
-class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
+class MCDwarfLineAddrFragment : public MCEncodedFragment {
/// The value of the
diff erence between the two line numbers
/// between two .loc dwarf directives.
int64_t LineDelta;
@@ -594,8 +616,8 @@ class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
public:
MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta)
- : MCEncodedFragmentWithFixups<8, 1>(FT_Dwarf, false),
- LineDelta(LineDelta), AddrDelta(&AddrDelta) {}
+ : MCEncodedFragment(FT_Dwarf, false), LineDelta(LineDelta),
+ AddrDelta(&AddrDelta) {}
int64_t getLineDelta() const { return LineDelta; }
@@ -606,15 +628,14 @@ class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
}
};
-class MCDwarfCallFrameFragment : public MCEncodedFragmentWithFixups<8, 1> {
+class MCDwarfCallFrameFragment : public MCEncodedFragment {
/// The expression for the
diff erence of the two symbols that
/// make up the address delta between two .cfi_* dwarf directives.
const MCExpr *AddrDelta;
public:
MCDwarfCallFrameFragment(const MCExpr &AddrDelta)
- : MCEncodedFragmentWithFixups<8, 1>(FT_DwarfFrame, false),
- AddrDelta(&AddrDelta) {}
+ : MCEncodedFragment(FT_DwarfFrame, false), AddrDelta(&AddrDelta) {}
const MCExpr &getAddrDelta() const { return *AddrDelta; }
void setAddrDelta(const MCExpr *E) { AddrDelta = E; }
@@ -642,13 +663,12 @@ class MCSymbolIdFragment : public MCFragment {
/// Fragment representing the binary annotations produced by the
/// .cv_inline_linetable directive.
-class MCCVInlineLineTableFragment : public MCFragment {
+class MCCVInlineLineTableFragment : public MCEncodedFragment {
unsigned SiteFuncId;
unsigned StartFileId;
unsigned StartLineNum;
const MCSymbol *FnStartSym;
const MCSymbol *FnEndSym;
- SmallString<8> Contents;
/// CodeViewContext has the real knowledge about this format, so let it access
/// our members.
@@ -658,23 +678,20 @@ class MCCVInlineLineTableFragment : public MCFragment {
MCCVInlineLineTableFragment(unsigned SiteFuncId, unsigned StartFileId,
unsigned StartLineNum, const MCSymbol *FnStartSym,
const MCSymbol *FnEndSym)
- : MCFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId),
+ : MCEncodedFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId),
StartFileId(StartFileId), StartLineNum(StartLineNum),
FnStartSym(FnStartSym), FnEndSym(FnEndSym) {}
const MCSymbol *getFnStartSym() const { return FnStartSym; }
const MCSymbol *getFnEndSym() const { return FnEndSym; }
- SmallString<8> &getContents() { return Contents; }
- const SmallString<8> &getContents() const { return Contents; }
-
static bool classof(const MCFragment *F) {
return F->getKind() == MCFragment::FT_CVInlineLines;
}
};
/// Fragment representing the .cv_def_range directive.
-class MCCVDefRangeFragment : public MCEncodedFragmentWithFixups<32, 4> {
+class MCCVDefRangeFragment : public MCEncodedFragment {
ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges;
StringRef FixedSizePortion;
@@ -686,8 +703,9 @@ class MCCVDefRangeFragment : public MCEncodedFragmentWithFixups<32, 4> {
MCCVDefRangeFragment(
ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges,
StringRef FixedSizePortion)
- : MCEncodedFragmentWithFixups<32, 4>(FT_CVDefRange, false),
- Ranges(Ranges), FixedSizePortion(FixedSizePortion) {}
+ : MCEncodedFragment(FT_CVDefRange, false),
+ Ranges(Ranges.begin(), Ranges.end()),
+ FixedSizePortion(FixedSizePortion) {}
ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> getRanges() const {
return Ranges;
@@ -739,15 +757,14 @@ class MCBoundaryAlignFragment : public MCFragment {
}
};
-class MCPseudoProbeAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
+class MCPseudoProbeAddrFragment : public MCEncodedFragment {
/// The expression for the
diff erence of the two symbols that
/// make up the address delta between two .pseudoprobe directives.
const MCExpr *AddrDelta;
public:
MCPseudoProbeAddrFragment(const MCExpr *AddrDelta)
- : MCEncodedFragmentWithFixups<8, 1>(FT_PseudoProbe, false),
- AddrDelta(AddrDelta) {}
+ : MCEncodedFragment(FT_PseudoProbe, false), AddrDelta(AddrDelta) {}
const MCExpr &getAddrDelta() const { return *AddrDelta; }
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 995a68e5548c8..a9155935968b3 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -606,12 +606,14 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
case MCFragment::FT_Data:
++stats::EmittedDataFragments;
- OS << cast<MCDataFragment>(F).getContents();
+ OS << StringRef(cast<MCDataFragment>(F).getContents().data(),
+ cast<MCDataFragment>(F).getContents().size());
break;
case MCFragment::FT_Relaxable:
++stats::EmittedRelaxableFragments;
- OS << cast<MCRelaxableFragment>(F).getContents();
+ OS << StringRef(cast<MCRelaxableFragment>(F).getContents().data(),
+ cast<MCRelaxableFragment>(F).getContents().size());
break;
case MCFragment::FT_Fill: {
@@ -691,7 +693,7 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
case MCFragment::FT_LEB: {
const MCLEBFragment &LF = cast<MCLEBFragment>(F);
- OS << LF.getContents();
+ OS << StringRef(LF.getContents().data(), LF.getContents().size());
break;
}
@@ -721,27 +723,27 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
case MCFragment::FT_Dwarf: {
const MCDwarfLineAddrFragment &OF = cast<MCDwarfLineAddrFragment>(F);
- OS << OF.getContents();
+ OS << StringRef(OF.getContents().data(), OF.getContents().size());
break;
}
case MCFragment::FT_DwarfFrame: {
const MCDwarfCallFrameFragment &CF = cast<MCDwarfCallFrameFragment>(F);
- OS << CF.getContents();
+ OS << StringRef(CF.getContents().data(), CF.getContents().size());
break;
}
case MCFragment::FT_CVInlineLines: {
const auto &OF = cast<MCCVInlineLineTableFragment>(F);
- OS << OF.getContents();
+ OS << StringRef(OF.getContents().data(), OF.getContents().size());
break;
}
case MCFragment::FT_CVDefRange: {
const auto &DRF = cast<MCCVDefRangeFragment>(F);
- OS << DRF.getContents();
+ OS << StringRef(DRF.getContents().data(), DRF.getContents().size());
break;
}
case MCFragment::FT_PseudoProbe: {
const MCPseudoProbeAddrFragment &PF = cast<MCPseudoProbeAddrFragment>(F);
- OS << PF.getContents();
+ OS << StringRef(PF.getContents().data(), PF.getContents().size());
break;
}
}
@@ -993,10 +995,11 @@ bool MCAssembler::relaxInstruction(MCRelaxableFragment &F) {
// Encode the new instruction.
F.setInst(Relaxed);
- F.getFixups().clear();
- F.getContents().clear();
- getEmitter().encodeInstruction(Relaxed, F.getContents(), F.getFixups(),
- *F.getSubtargetInfo());
+ SmallVector<char, 16> Data;
+ SmallVector<MCFixup, 1> Fixups;
+ getEmitter().encodeInstruction(Relaxed, Data, Fixups, *F.getSubtargetInfo());
+ F.setContents(Data);
+ F.setFixups(Fixups);
return true;
}
@@ -1004,8 +1007,7 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) {
const unsigned OldSize = static_cast<unsigned>(LF.getContents().size());
unsigned PadTo = OldSize;
int64_t Value;
- SmallVectorImpl<char> &Data = LF.getContents();
- LF.getFixups().clear();
+ LF.clearFixups();
// Use evaluateKnownAbsolute for Mach-O as a hack: .subsections_via_symbols
// requires that .uleb128 A-B is foldable where A and B reside in
diff erent
// fragments. This is used by __gcc_except_table.
@@ -1026,17 +1028,18 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) {
if (UseZeroPad)
Value = 0;
}
- Data.clear();
- raw_svector_ostream OSE(Data);
+ uint8_t Data[16];
+ size_t Size = 0;
// The compiler can generate EH table assembly that is impossible to assemble
// without either adding padding to an LEB fragment or adding extra padding
// to a later alignment fragment. To accommodate such tables, relaxation can
// only increase an LEB fragment size here, not decrease it. See PR35809.
if (LF.isSigned())
- encodeSLEB128(Value, OSE, PadTo);
+ Size = encodeSLEB128(Value, Data, PadTo);
else
- encodeULEB128(Value, OSE, PadTo);
- return OldSize != LF.getContents().size();
+ Size = encodeULEB128(Value, Data, PadTo);
+ LF.setContents({reinterpret_cast<char *>(Data), Size});
+ return OldSize != Size;
}
/// Check if the branch crosses the boundary.
@@ -1106,19 +1109,19 @@ bool MCAssembler::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF) {
return WasRelaxed;
MCContext &Context = getContext();
- uint64_t OldSize = DF.getContents().size();
+ auto OldSize = DF.getContents().size();
int64_t AddrDelta;
bool Abs = DF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this);
assert(Abs && "We created a line delta with an invalid expression");
(void)Abs;
int64_t LineDelta;
LineDelta = DF.getLineDelta();
- SmallVectorImpl<char> &Data = DF.getContents();
- Data.clear();
- DF.getFixups().clear();
+ SmallVector<char, 8> Data;
MCDwarfLineAddr::encode(Context, getDWARFLinetableParams(), LineDelta,
AddrDelta, Data);
+ DF.setContents(Data);
+ DF.clearFixups();
return OldSize != Data.size();
}
@@ -1137,12 +1140,11 @@ bool MCAssembler::relaxDwarfCallFrameFragment(MCDwarfCallFrameFragment &DF) {
return false;
}
- SmallVectorImpl<char> &Data = DF.getContents();
- uint64_t OldSize = Data.size();
- Data.clear();
- DF.getFixups().clear();
-
+ auto OldSize = DF.getContents().size();
+ SmallVector<char, 8> Data;
MCDwarfFrameEmitter::encodeAdvanceLoc(Context, Value, Data);
+ DF.setContents(Data);
+ DF.clearFixups();
return OldSize != Data.size();
}
@@ -1172,13 +1174,13 @@ bool MCAssembler::relaxPseudoProbeAddr(MCPseudoProbeAddrFragment &PF) {
bool Abs = PF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this);
assert(Abs && "We created a pseudo probe with an invalid expression");
(void)Abs;
- SmallVectorImpl<char> &Data = PF.getContents();
- Data.clear();
+ SmallVector<char, 8> Data;
raw_svector_ostream OSE(Data);
- PF.getFixups().clear();
// AddrDelta is a signed integer
encodeSLEB128(AddrDelta, OSE, OldSize);
+ PF.setContents(Data);
+ PF.clearFixups();
return OldSize != Data.size();
}
diff --git a/llvm/lib/MC/MCCodeView.cpp b/llvm/lib/MC/MCCodeView.cpp
index e8f04271e84ce..5d7914396e09f 100644
--- a/llvm/lib/MC/MCCodeView.cpp
+++ b/llvm/lib/MC/MCCodeView.cpp
@@ -510,8 +510,7 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm,
MCCVFunctionInfo *SiteInfo = getCVFunctionInfo(Frag.SiteFuncId);
- SmallVectorImpl<char> &Buffer = Frag.getContents();
- Buffer.clear(); // Clear old contents if we went through relaxation.
+ SmallVector<char, 0> Buffer;
for (const MCCVLoc &Loc : Locs) {
// Exit early if our line table would produce an oversized InlineSiteSym
// record. Account for the ChangeCodeLength annotation emitted after the
@@ -604,15 +603,14 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm,
compressAnnotation(BinaryAnnotationsOpCode::ChangeCodeLength, Buffer);
compressAnnotation(std::min(EndSymLength, LocAfterLength), Buffer);
+ Frag.setContents(Buffer);
}
void CodeViewContext::encodeDefRange(const MCAssembler &Asm,
MCCVDefRangeFragment &Frag) {
MCContext &Ctx = Asm.getContext();
- SmallVectorImpl<char> &Contents = Frag.getContents();
- Contents.clear();
- SmallVectorImpl<MCFixup> &Fixups = Frag.getFixups();
- Fixups.clear();
+ SmallVector<char, 0> Contents;
+ SmallVector<MCFixup, 0> Fixups;
raw_svector_ostream OS(Contents);
// Compute all the sizes up front.
@@ -692,4 +690,7 @@ void CodeViewContext::encodeDefRange(const MCAssembler &Asm,
GapStartOffset += GapSize + RangeSize;
}
}
+
+ Frag.setContents(Contents);
+ Frag.setFixups(Fixups);
}
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 1fecde4be03cc..e84b91c5c41b0 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -448,11 +448,14 @@ void MCELFStreamer::emitInstToData(const MCInst &Inst,
// Emit instruction directly into data fragment.
size_t FixupStartIndex = DF->getFixups().size();
size_t CodeOffset = DF->getContents().size();
- Assembler.getEmitter().encodeInstruction(Inst, DF->getContents(),
- DF->getFixups(), STI);
-
- auto Fixups = MutableArrayRef(DF->getFixups()).slice(FixupStartIndex);
- for (auto &Fixup : Fixups) {
+ SmallVector<MCFixup, 1> Fixups;
+ Assembler.getEmitter().encodeInstruction(Inst, DF->getContentsForAppending(),
+ Fixups, STI);
+ DF->doneAppending();
+ if (!Fixups.empty())
+ DF->appendFixups(Fixups);
+
+ for (auto &Fixup : MutableArrayRef(DF->getFixups()).slice(FixupStartIndex)) {
Fixup.setOffset(Fixup.getOffset() + CodeOffset);
if (Fixup.isLinkerRelaxable()) {
DF->setLinkerRelaxable();
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index 7dd3b501757b7..5cf47e3325ff2 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -19,66 +19,22 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
+#include <type_traits>
#include <utility>
using namespace llvm;
+static_assert(std::is_trivially_destructible_v<MCDataFragment>,
+ "fragment classes must be trivially destructible");
+
MCFragment::MCFragment(FragmentType Kind, bool HasInstructions)
: Kind(Kind), HasInstructions(HasInstructions), AlignToBundleEnd(false),
LinkerRelaxable(false), AllowAutoPadding(false) {}
-void MCFragment::destroy() {
- switch (Kind) {
- case FT_Align:
- cast<MCAlignFragment>(this)->~MCAlignFragment();
- return;
- case FT_Data:
- cast<MCDataFragment>(this)->~MCDataFragment();
- return;
- case FT_Fill:
- cast<MCFillFragment>(this)->~MCFillFragment();
- return;
- case FT_Nops:
- cast<MCNopsFragment>(this)->~MCNopsFragment();
- return;
- case FT_Relaxable:
- cast<MCRelaxableFragment>(this)->~MCRelaxableFragment();
- return;
- case FT_Org:
- cast<MCOrgFragment>(this)->~MCOrgFragment();
- return;
- case FT_Dwarf:
- cast<MCDwarfLineAddrFragment>(this)->~MCDwarfLineAddrFragment();
- return;
- case FT_DwarfFrame:
- cast<MCDwarfCallFrameFragment>(this)->~MCDwarfCallFrameFragment();
- return;
- case FT_LEB:
- cast<MCLEBFragment>(this)->~MCLEBFragment();
- return;
- case FT_BoundaryAlign:
- cast<MCBoundaryAlignFragment>(this)->~MCBoundaryAlignFragment();
- return;
- case FT_SymbolId:
- cast<MCSymbolIdFragment>(this)->~MCSymbolIdFragment();
- return;
- case FT_CVInlineLines:
- cast<MCCVInlineLineTableFragment>(this)->~MCCVInlineLineTableFragment();
- return;
- case FT_CVDefRange:
- cast<MCCVDefRangeFragment>(this)->~MCCVDefRangeFragment();
- return;
- case FT_PseudoProbe:
- cast<MCPseudoProbeAddrFragment>(this)->~MCPseudoProbeAddrFragment();
- return;
- }
-}
-
const MCSymbol *MCFragment::getAtom() const {
return cast<MCSectionMachO>(Parent)->getAtom(LayoutOrder);
}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void MCFragment::dump() const {
raw_ostream &OS = errs();
@@ -131,7 +87,7 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
const auto *F = cast<MCDataFragment>(this);
if (F->isLinkerRelaxable())
OS << " LinkerRelaxable";
- const SmallVectorImpl<char> &Contents = F->getContents();
+ auto Contents = F->getContents();
OS << " Size:" << Contents.size() << " [";
for (unsigned i = 0, e = Contents.size(); i != e; ++i) {
if (i) OS << ",";
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 3b2bb594675f0..62b261a1268bc 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -63,28 +63,14 @@ void MCObjectStreamer::resolvePendingFixups() {
PendingFixup.Fixup.setOffset(PendingFixup.Sym->getOffset() +
PendingFixup.Fixup.getOffset());
- // If the location symbol to relocate is in MCEncodedFragmentWithFixups,
+ // If the location symbol to relocate is in MCEncodedFragment,
// put the Fixup into location symbol's fragment. Otherwise
// put into PendingFixup.DF
MCFragment *SymFragment = PendingFixup.Sym->getFragment();
- switch (SymFragment->getKind()) {
- case MCFragment::FT_Relaxable:
- case MCFragment::FT_Dwarf:
- case MCFragment::FT_PseudoProbe:
- cast<MCEncodedFragmentWithFixups<8, 1>>(SymFragment)
- ->getFixups()
- .push_back(PendingFixup.Fixup);
- break;
- case MCFragment::FT_Data:
- case MCFragment::FT_CVDefRange:
- cast<MCEncodedFragmentWithFixups<32, 4>>(SymFragment)
- ->getFixups()
- .push_back(PendingFixup.Fixup);
- break;
- default:
+ if (auto *F = dyn_cast<MCEncodedFragment>(SymFragment))
+ F->addFixup(PendingFixup.Fixup);
+ else
PendingFixup.DF->addFixup(PendingFixup.Fixup);
- break;
- }
}
PendingFixups.clear();
}
@@ -398,10 +384,10 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst,
getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
auto CodeOffset = DF->getContents().size();
- for (MCFixup &Fixup : Fixups) {
+ for (MCFixup &Fixup : Fixups)
Fixup.setOffset(Fixup.getOffset() + CodeOffset);
- DF->addFixup(Fixup);
- }
+ if (!Fixups.empty())
+ DF->appendFixups(Fixups);
DF->setHasInstructions(STI);
DF->appendContents(Code);
}
@@ -414,8 +400,11 @@ void MCObjectStreamer::emitInstToFragment(const MCInst &Inst,
getContext().allocFragment<MCRelaxableFragment>(Inst, STI);
insert(IF);
- getAssembler().getEmitter().encodeInstruction(Inst, IF->getContents(),
- IF->getFixups(), STI);
+ SmallVector<MCFixup, 1> Fixups;
+ getAssembler().getEmitter().encodeInstruction(
+ Inst, IF->getContentsForAppending(), Fixups, STI);
+ IF->doneAppending();
+ IF->appendFixups(Fixups);
}
#ifndef NDEBUG
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index 1b766d3fad8b1..09d09e8f997e6 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -11,6 +11,7 @@
#include "llvm/Config/llvm-config.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
@@ -36,10 +37,13 @@ MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) {
bool MCSection::hasEnded() const { return End && End->isInSection(); }
MCSection::~MCSection() {
+ // If ~MCRelaxableFragment becomes trivial (no longer store a MCInst member),
+ // this dtor can be made empty.
for (auto &[_, Chain] : Subsections) {
for (MCFragment *X = Chain.Head, *Y; X; X = Y) {
Y = X->Next;
- X->destroy();
+ if (auto *F = dyn_cast<MCRelaxableFragment>(X))
+ F->~MCRelaxableFragment();
}
}
}
@@ -88,3 +92,39 @@ LLVM_DUMP_METHOD void MCSection::dump(
}
}
#endif
+
+void MCEncodedFragment::setContents(ArrayRef<char> Contents) {
+ auto &S = getParent()->ContentStorage;
+ if (ContentStart + Contents.size() > ContentEnd) {
+ ContentStart = S.size();
+ S.resize_for_overwrite(S.size() + Contents.size());
+ }
+ ContentEnd = ContentStart + Contents.size();
+ llvm::copy(Contents, S.begin() + ContentStart);
+}
+
+void MCEncodedFragment::addFixup(MCFixup Fixup) { appendFixups({Fixup}); }
+
+void MCEncodedFragment::appendFixups(ArrayRef<MCFixup> Fixups) {
+ auto &S = getParent()->FixupStorage;
+ if (LLVM_UNLIKELY(FixupEnd != S.size())) {
+ // Move the elements to the end. Reserve space to avoid invalidating
+ // S.begin()+I for `append`.
+ auto Size = FixupEnd - FixupStart;
+ auto I = std::exchange(FixupStart, S.size());
+ S.reserve(S.size() + Size);
+ S.append(S.begin() + I, S.begin() + I + Size);
+ }
+ S.append(Fixups.begin(), Fixups.end());
+ FixupEnd = S.size();
+}
+
+void MCEncodedFragment::setFixups(ArrayRef<MCFixup> Fixups) {
+ auto &S = getParent()->FixupStorage;
+ if (FixupStart + Fixups.size() > FixupEnd) {
+ FixupStart = S.size();
+ S.resize_for_overwrite(S.size() + Fixups.size());
+ }
+ FixupEnd = FixupStart + Fixups.size();
+ llvm::copy(Fixups, S.begin() + FixupStart);
+}
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 2e671e0685c2e..eb7cb7d634311 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -807,8 +807,8 @@ uint64_t MachObjectWriter::writeObject() {
MCSection *CGProfileSection = getContext().getMachOSection(
"__LLVM", "__cg_profile", 0, SectionKind::getMetadata());
auto &Frag = cast<MCDataFragment>(*CGProfileSection->begin());
- Frag.getContents().clear();
- raw_svector_ostream OS(Frag.getContents());
+ Frag.clearContents();
+ raw_svector_ostream OS(Frag.getContentsForAppending());
for (const MCObjectWriter::CGProfileEntry &CGPE : CGProfile) {
uint32_t FromIndex = CGPE.From->getSymbol().getIndex();
uint32_t ToIndex = CGPE.To->getSymbol().getIndex();
@@ -816,6 +816,7 @@ uint64_t MachObjectWriter::writeObject() {
support::endian::write(OS, ToIndex, W.Endian);
support::endian::write(OS, CGPE.Count, W.Endian);
}
+ Frag.doneAppending();
}
unsigned NumSections = Asm.end() - Asm.begin();
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index a822a80b0af42..c2ef430984ed4 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -1070,7 +1070,7 @@ uint64_t WinCOFFWriter::writeObject() {
auto *Sec = getContext().getCOFFSection(".llvm_addrsig",
COFF::IMAGE_SCN_LNK_REMOVE);
auto *Frag = cast<MCDataFragment>(Sec->curFragList()->Head);
- raw_svector_ostream OS(Frag->getContents());
+ raw_svector_ostream OS(Frag->getContentsForAppending());
for (const MCSymbol *S : OWriter.AddrsigSyms) {
if (!S->isRegistered())
continue;
@@ -1085,6 +1085,7 @@ uint64_t WinCOFFWriter::writeObject() {
"executePostLayoutBinding!");
encodeULEB128(SectionMap[TargetSection]->Symbol->getIndex(), OS);
}
+ Frag->doneAppending();
}
// Create the contents of the .llvm.call-graph-profile section.
@@ -1092,7 +1093,7 @@ uint64_t WinCOFFWriter::writeObject() {
auto *Sec = getContext().getCOFFSection(".llvm.call-graph-profile",
COFF::IMAGE_SCN_LNK_REMOVE);
auto *Frag = cast<MCDataFragment>(Sec->curFragList()->Head);
- raw_svector_ostream OS(Frag->getContents());
+ raw_svector_ostream OS(Frag->getContentsForAppending());
for (const auto &CGPE : OWriter.getCGProfile()) {
uint32_t FromIndex = CGPE.From->getSymbol().getIndex();
uint32_t ToIndex = CGPE.To->getSymbol().getIndex();
@@ -1100,6 +1101,7 @@ uint64_t WinCOFFWriter::writeObject() {
support::endian::write(OS, ToIndex, W.Endian);
support::endian::write(OS, CGPE.Count, W.Endian);
}
+ Frag->doneAppending();
}
assignFileOffsets();
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index f7bb33d2790a1..64192ed6632de 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -297,9 +297,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
int64_t LineDelta = DF.getLineDelta();
const MCExpr &AddrDelta = DF.getAddrDelta();
- SmallVectorImpl<char> &Data = DF.getContents();
- SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
- size_t OldSize = Data.size();
+ SmallVector<MCFixup, 1> Fixups;
+ size_t OldSize = DF.getContents().size();
int64_t Value;
if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -308,8 +307,7 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
assert(IsAbsolute && "CFA with invalid expression");
(void)IsAbsolute;
- Data.clear();
- Fixups.clear();
+ SmallVector<char> Data;
raw_svector_ostream OS(Data);
// INT64_MAX is a signal that this is actually a DW_LNE_end_sequence.
@@ -354,6 +352,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
OS << uint8_t(dwarf::DW_LNS_copy);
}
+ DF.setContents(Data);
+ DF.setFixups(Fixups);
WasRelaxed = OldSize != Data.size();
return true;
}
@@ -361,9 +361,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
bool &WasRelaxed) const {
const MCExpr &AddrDelta = DF.getAddrDelta();
- SmallVectorImpl<char> &Data = DF.getContents();
- SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
- size_t OldSize = Data.size();
+ SmallVector<MCFixup, 2> Fixups;
+ size_t OldSize = DF.getContents().size();
int64_t Value;
if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -372,14 +371,12 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
assert(IsAbsolute && "CFA with invalid expression");
(void)IsAbsolute;
- Data.clear();
- Fixups.clear();
- raw_svector_ostream OS(Data);
-
assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 &&
"expected 1-byte alignment");
if (Value == 0) {
- WasRelaxed = OldSize != Data.size();
+ DF.clearContents();
+ DF.clearFixups();
+ WasRelaxed = OldSize != DF.getContents().size();
return true;
}
@@ -391,6 +388,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(FK)));
};
+ SmallVector<char, 8> Data;
+ raw_svector_ostream OS(Data);
if (isUIntN(6, Value)) {
OS << uint8_t(dwarf::DW_CFA_advance_loc);
AddFixups(0, getRelocPairForSize(6));
@@ -409,6 +408,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
} else {
llvm_unreachable("unsupported CFA encoding");
}
+ DF.setContents(Data);
+ DF.setFixups(Fixups);
WasRelaxed = OldSize != Data.size();
return true;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 952744b549811..75ef861d58a1c 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -274,17 +274,16 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
int64_t LineDelta = DF.getLineDelta();
const MCExpr &AddrDelta = DF.getAddrDelta();
- SmallVectorImpl<char> &Data = DF.getContents();
- SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
- size_t OldSize = Data.size();
+ SmallVector<MCFixup, 1> Fixups;
+ size_t OldSize = DF.getContents().size();
int64_t Value;
[[maybe_unused]] bool IsAbsolute =
AddrDelta.evaluateKnownAbsolute(Value, *Asm);
assert(IsAbsolute && "CFA with invalid expression");
- Data.clear();
Fixups.clear();
+ SmallVector<char> Data;
raw_svector_ostream OS(Data);
// INT64_MAX is a signal that this is actually a DW_LNE_end_sequence.
@@ -329,6 +328,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
OS << uint8_t(dwarf::DW_LNS_copy);
}
+ DF.setContents(Data);
+ DF.setFixups(Fixups);
WasRelaxed = OldSize != Data.size();
return true;
}
@@ -336,9 +337,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
bool &WasRelaxed) const {
const MCExpr &AddrDelta = DF.getAddrDelta();
- SmallVectorImpl<char> &Data = DF.getContents();
- SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
- size_t OldSize = Data.size();
+ SmallVector<MCFixup, 2> Fixups;
+ size_t OldSize = DF.getContents().size();
int64_t Value;
if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -347,14 +347,12 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
AddrDelta.evaluateKnownAbsolute(Value, *Asm);
assert(IsAbsolute && "CFA with invalid expression");
- Data.clear();
- Fixups.clear();
- raw_svector_ostream OS(Data);
-
assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 &&
"expected 1-byte alignment");
if (Value == 0) {
- WasRelaxed = OldSize != Data.size();
+ DF.clearContents();
+ DF.clearFixups();
+ WasRelaxed = OldSize != DF.getContents().size();
return true;
}
@@ -365,6 +363,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(Fixup)));
};
+ SmallVector<char, 8> Data;
+ raw_svector_ostream OS(Data);
if (isUIntN(6, Value)) {
OS << uint8_t(dwarf::DW_CFA_advance_loc);
AddFixups(0, {ELF::R_RISCV_SET6, ELF::R_RISCV_SUB6});
@@ -383,6 +383,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
} else {
llvm_unreachable("unsupported CFA encoding");
}
+ DF.setContents(Data);
+ DF.setFixups(Fixups);
WasRelaxed = OldSize != Data.size();
return true;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 361d7b488d048..77e2011361162 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -847,7 +847,7 @@ bool X86AsmBackend::padInstructionViaRelaxation(MCRelaxableFragment &RF,
return false;
RF.setInst(Relaxed);
RF.setContents(Code);
- RF.getFixups() = Fixups;
+ RF.setFixups(Fixups);
RemainingSize -= Delta;
return true;
}
More information about the llvm-commits
mailing list