[llvm] MCFragment: Use trailing data for fixed-size part (PR #150846)
Fangrui Song via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 28 22:06:16 PDT 2025
https://github.com/MaskRay updated https://github.com/llvm/llvm-project/pull/150846
>From 94aaca75a3af64cd20e2848fb93de090519af135 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sun, 27 Jul 2025 12:11:56 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.5-bogner
---
llvm/include/llvm/MC/MCObjectStreamer.h | 10 ++
llvm/include/llvm/MC/MCSection.h | 40 +----
llvm/lib/MC/MCMachOStreamer.cpp | 5 +-
llvm/lib/MC/MCObjectStreamer.cpp | 137 ++++++++++++++----
llvm/lib/MC/MCWin64EH.cpp | 2 +
llvm/lib/MC/MachObjectWriter.cpp | 2 +-
.../Mips/MCTargetDesc/MipsTargetStreamer.cpp | 6 +
7 files changed, 134 insertions(+), 68 deletions(-)
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index bbbee15abf700..b6e24816d94db 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -52,6 +52,10 @@ class MCObjectStreamer : public MCStreamer {
DenseMap<const MCSymbol *, SmallVector<PendingAssignment, 1>>
pendingAssignments;
+ SmallVector<std::unique_ptr<char[]>, 0> FragStorage;
+ // Available bytes in the current block for trailing data or new fragments.
+ size_t FragSpace = 0;
+
void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &);
void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
@@ -84,9 +88,15 @@ class MCObjectStreamer : public MCStreamer {
// Add a fragment with a variable-size tail and start a new empty fragment.
void insert(MCFragment *F);
+ char *getCurFragEnd() const {
+ return reinterpret_cast<char *>(CurFrag + 1) + CurFrag->getFixedSize();
+ }
+ MCFragment *allocFragSpace(size_t Headroom);
// Add a new fragment to the current section without a variable-size tail.
void newFragment();
+ void ensureHeadroom(size_t Headroom);
+ void appendContents(ArrayRef<char> Contents);
void appendContents(size_t Num, char Elt);
void addFixup(const MCExpr *Value, MCFixupKind Kind);
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 7989310e5a8f2..1579fa6646c3c 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -93,8 +93,7 @@ class MCFragment {
// Track content and fixups for the fixed-size part as fragments are
// appended to the section. The content remains immutable, except when
// modified by applyFixup.
- uint32_t ContentStart = 0;
- uint32_t ContentEnd = 0;
+ uint32_t FixedSize = 0;
uint32_t FixupStart = 0;
uint32_t FixupEnd = 0;
@@ -205,18 +204,6 @@ class MCFragment {
//== Content-related functions manage parent's storage using ContentStart and
// ContentSize.
- // Get a SmallVector reference. The caller should call doneAppending to update
- // `ContentEnd`.
- SmallVectorImpl<char> &getContentsForAppending();
- void doneAppending();
- void appendContents(ArrayRef<char> Contents) {
- getContentsForAppending().append(Contents.begin(), Contents.end());
- doneAppending();
- }
- void appendContents(size_t Num, char Elt) {
- getContentsForAppending().append(Num, Elt);
- doneAppending();
- }
MutableArrayRef<char> getContents();
ArrayRef<char> getContents() const;
@@ -225,10 +212,10 @@ class MCFragment {
MutableArrayRef<char> getVarContents();
ArrayRef<char> getVarContents() const;
- size_t getFixedSize() const { return ContentEnd - ContentStart; }
+ size_t getFixedSize() const { return FixedSize; }
size_t getVarSize() const { return VarContentEnd - VarContentStart; }
size_t getSize() const {
- return ContentEnd - ContentStart + (VarContentEnd - VarContentStart);
+ return FixedSize + (VarContentEnd - VarContentStart);
}
//== Fixup-related functions manage parent's storage using FixupStart and
@@ -651,28 +638,11 @@ class LLVM_ABI MCSection {
bool isBssSection() const { return IsBss; }
};
-inline SmallVectorImpl<char> &MCFragment::getContentsForAppending() {
- SmallVectorImpl<char> &S = getParent()->ContentStorage;
- if (LLVM_UNLIKELY(ContentEnd != S.size())) {
- // Move the elements to the end. Reserve space to avoid invalidating
- // S.begin()+I for `append`.
- auto Size = ContentEnd - ContentStart;
- auto I = std::exchange(ContentStart, S.size());
- S.reserve(S.size() + Size);
- S.append(S.begin() + I, S.begin() + I + Size);
- }
- return S;
-}
-inline void MCFragment::doneAppending() {
- ContentEnd = getParent()->ContentStorage.size();
-}
inline MutableArrayRef<char> MCFragment::getContents() {
- return MutableArrayRef(getParent()->ContentStorage)
- .slice(ContentStart, ContentEnd - ContentStart);
+ return {reinterpret_cast<char *>(this + 1), FixedSize};
}
inline ArrayRef<char> MCFragment::getContents() const {
- return ArrayRef(getParent()->ContentStorage)
- .slice(ContentStart, ContentEnd - ContentStart);
+ return {reinterpret_cast<const char *>(this + 1), FixedSize};
}
inline MutableArrayRef<char> MCFragment::getVarContents() {
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index 107466912d090..1c5abd6ddacc7 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -484,7 +484,8 @@ void MCMachOStreamer::finalizeCGProfile() {
// For each entry, reserve space for 2 32-bit indices and a 64-bit count.
size_t SectionBytes =
W.getCGProfile().size() * (2 * sizeof(uint32_t) + sizeof(uint64_t));
- (*CGProfileSection->begin()).appendContents(SectionBytes, 0);
+ (*CGProfileSection->begin())
+ .setVarContents(std::vector<char>(SectionBytes, 0));
}
MCStreamer *llvm::createMachOStreamer(MCContext &Context,
@@ -520,5 +521,5 @@ void MCMachOStreamer::createAddrSigSection() {
// (instead of emitting a zero-sized section) so these relocations are
// technically valid, even though we don't expect these relocations to
// actually be applied by the linker.
- Frag->appendContents(8, 0);
+ Frag->setVarContents(std::vector<char>(8, 0));
}
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 9c7b05bd9a45a..28293501fb8e6 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -46,23 +46,84 @@ MCAssembler *MCObjectStreamer::getAssemblerPtr() {
return nullptr;
}
+constexpr size_t FragChunkSize = 16384;
+// Ensure the new fragment can at least at least a few bytes.
+constexpr size_t NewFragHeadroom = 8;
+
+static_assert(NewFragHeadroom >= alignof(MCFragment));
+static_assert(FragChunkSize >= sizeof(MCFragment) + NewFragHeadroom);
+
+MCFragment *MCObjectStreamer::allocFragSpace(size_t Headroom) {
+ auto Size = std::max(FragChunkSize, sizeof(MCFragment) + Headroom);
+ FragSpace = Size - sizeof(MCFragment);
+ auto Chunk = std::unique_ptr<char[]>(new char[Size]);
+ auto *F = reinterpret_cast<MCFragment *>(Chunk.get());
+ FragStorage.push_back(std::move(Chunk));
+ return F;
+}
+
void MCObjectStreamer::newFragment() {
- addFragment(getContext().allocFragment<MCFragment>());
+ MCFragment *F;
+ if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
+ auto End = size_t(getCurFragEnd());
+ F = reinterpret_cast<MCFragment *>(
+ alignToPowerOf2(End, alignof(MCFragment)));
+ FragSpace -= size_t(F) - End + sizeof(MCFragment);
+ } else {
+ F = allocFragSpace(0);
+ }
+ new (F) MCFragment();
+ addFragment(F);
+}
+
+void MCObjectStreamer::ensureHeadroom(size_t Headroom) {
+ if (Headroom <= FragSpace)
+ return;
+ auto *F = allocFragSpace(Headroom);
+ new (F) MCFragment();
+ addFragment(F);
}
-void MCObjectStreamer::insert(MCFragment *F) {
- assert(F->getKind() != MCFragment::FT_Data &&
+void MCObjectStreamer::insert(MCFragment *Frag) {
+ assert(Frag->getKind() != MCFragment::FT_Data &&
"F should have a variable-size tail");
+ // Allocate an empty fragment, potentially reusing the space associated with
+ // CurFrag.
+ MCFragment *F;
+ if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
+ auto End = size_t(getCurFragEnd());
+ F = reinterpret_cast<MCFragment *>(
+ alignToPowerOf2(End, alignof(MCFragment)));
+ FragSpace -= size_t(F) - End + sizeof(MCFragment);
+ } else {
+ F = allocFragSpace(0);
+ }
+
+ // Add Frag, which destroys CurFrag and the associated space.
+ addFragment(Frag);
+ new (F) MCFragment();
+ // Add the empty fragment, which restores CurFrag and the associated space.
addFragment(F);
- newFragment();
+}
+
+void MCObjectStreamer::appendContents(ArrayRef<char> Contents) {
+ ensureHeadroom(Contents.size());
+ assert(FragSpace >= Contents.size());
+ llvm::copy(Contents, getCurFragEnd());
+ CurFrag->FixedSize += Contents.size();
+ FragSpace -= Contents.size();
}
void MCObjectStreamer::appendContents(size_t Num, char Elt) {
- CurFrag->appendContents(Num, Elt);
+ ensureHeadroom(Num);
+ MutableArrayRef<char> Data(getCurFragEnd(), Num);
+ llvm::fill(Data, Elt);
+ CurFrag->FixedSize += Num;
+ FragSpace -= Num;
}
void MCObjectStreamer::addFixup(const MCExpr *Value, MCFixupKind Kind) {
- CurFrag->addFixup(MCFixup::create(CurFrag->getFixedSize(), Value, Kind));
+ CurFrag->addFixup(MCFixup::create(getCurFragSize(), Value, Kind));
}
// As a compile-time optimization, avoid allocating and evaluating an MCExpr
@@ -111,6 +172,8 @@ void MCObjectStreamer::reset() {
}
EmitEHFrame = true;
EmitDebugFrame = false;
+ FragStorage.clear();
+ FragSpace = 0;
MCStreamer::reset();
}
@@ -139,7 +202,6 @@ void MCObjectStreamer::emitCFISections(bool EH, bool Debug, bool SFrame) {
void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
SMLoc Loc) {
MCStreamer::emitValueImpl(Value, Size, Loc);
- MCFragment *DF = getCurrentFragment();
MCDwarfLineEntry::make(this, getCurrentSectionOnly());
@@ -154,9 +216,9 @@ void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
emitIntValue(AbsValue, Size);
return;
}
- DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
- MCFixup::getDataKindForSize(Size)));
- DF->appendContents(Size, 0);
+ ensureHeadroom(Size);
+ addFixup(Value, MCFixup::getDataKindForSize(Size));
+ appendContents(Size, 0);
}
MCSymbol *MCObjectStreamer::emitCFILabel() {
@@ -190,7 +252,7 @@ void MCObjectStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) {
// section.
MCFragment *F = CurFrag;
Symbol->setFragment(F);
- Symbol->setOffset(F->getContents().size());
+ Symbol->setOffset(F->getFixedSize());
emitPendingAssignments(Symbol);
}
@@ -256,6 +318,21 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
F0 = CurFrag;
}
+ // CurFrag will be changed. To ensure that FragSpace remains connected with
+ // CurFrag, allocate an empty fragment and add it to the fragment list.
+ // (Subsections[I].second.Tail is disconnected with FragSpace.)
+ MCFragment *F;
+ if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
+ auto End = size_t(getCurFragEnd());
+ F = reinterpret_cast<MCFragment *>(
+ alignToPowerOf2(End, alignof(MCFragment)));
+ FragSpace -= size_t(F) - End + sizeof(MCFragment);
+ } else {
+ F = allocFragSpace(0);
+ }
+ new (F) MCFragment();
+ F->setParent(Section);
+
auto &Subsections = Section->Subsections;
size_t I = 0, E = Subsections.size();
while (I != E && Subsections[I].first < Subsection)
@@ -263,13 +340,16 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
// If the subsection number is not in the sorted Subsections list, create a
// new fragment list.
if (I == E || Subsections[I].first != Subsection) {
- auto *F = getContext().allocFragment<MCFragment>();
- F->setParent(Section);
Subsections.insert(Subsections.begin() + I,
{Subsection, MCSection::FragList{F, F}});
+ Section->CurFragList = &Subsections[I].second;
+ CurFrag = F;
+ } else {
+ Section->CurFragList = &Subsections[I].second;
+ CurFrag = Subsections[I].second.Tail;
+ // Ensure CurFrag is associated with FragSpace.
+ addFragment(F);
}
- Section->CurFragList = &Subsections[I].second;
- CurFrag = Section->CurFragList->Tail;
// Define the section symbol at subsection 0's initial fragment if required.
if (!NewSec)
@@ -340,11 +420,11 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst,
MCFragment *F = getCurrentFragment();
// Append the instruction to the data fragment.
- size_t CodeOffset = F->getContents().size();
+ size_t CodeOffset = getCurFragSize();
+ SmallString<16> Content;
SmallVector<MCFixup, 1> Fixups;
- getAssembler().getEmitter().encodeInstruction(
- Inst, F->getContentsForAppending(), Fixups, STI);
- F->doneAppending();
+ getAssembler().getEmitter().encodeInstruction(Inst, Content, Fixups, STI);
+ appendContents(Content);
F->setHasInstructions(STI);
if (Fixups.empty())
@@ -352,19 +432,17 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst,
bool MarkedLinkerRelaxable = false;
for (auto &Fixup : Fixups) {
Fixup.setOffset(Fixup.getOffset() + CodeOffset);
- if (!Fixup.isLinkerRelaxable() || MarkedLinkerRelaxable)
+ if (!Fixup.isLinkerRelaxable())
continue;
- MarkedLinkerRelaxable = true;
- // Set the fragment's order within the subsection for use by
- // MCAssembler::relaxAlign.
- auto *Sec = F->getParent();
- if (!Sec->isLinkerRelaxable())
- Sec->setLinkerRelaxable();
+ F->setLinkerRelaxable();
// Do not add data after a linker-relaxable instruction. The difference
// between a new label and a label at or before the linker-relaxable
// instruction cannot be resolved at assemble-time.
- F->setLinkerRelaxable();
- newFragment();
+ if (!MarkedLinkerRelaxable) {
+ MarkedLinkerRelaxable = true;
+ getCurrentSectionOnly()->setLinkerRelaxable();
+ newFragment();
+ }
}
F->appendFixups(Fixups);
}
@@ -538,8 +616,7 @@ void MCObjectStreamer::emitCVFileChecksumOffsetDirective(unsigned FileNo) {
void MCObjectStreamer::emitBytes(StringRef Data) {
MCDwarfLineEntry::make(this, getCurrentSectionOnly());
- MCFragment *DF = getCurrentFragment();
- DF->appendContents(ArrayRef(Data.data(), Data.size()));
+ appendContents(ArrayRef(Data.data(), Data.size()));
}
void MCObjectStreamer::emitValueToAlignment(Align Alignment, int64_t Fill,
diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp
index 72a8dd7031198..e212663d53a79 100644
--- a/llvm/lib/MC/MCWin64EH.cpp
+++ b/llvm/lib/MC/MCWin64EH.cpp
@@ -318,6 +318,8 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
// Emit the epilog instructions.
if (EnableUnwindV2) {
+ OS->ensureHeadroom(info->EpilogMap.size() * 2);
+
bool IsLast = true;
for (const auto &Epilog : llvm::reverse(info->EpilogMap)) {
if (IsLast) {
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 7b5c3c00f2519..e87696a93cddd 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -806,7 +806,7 @@ uint64_t MachObjectWriter::writeObject() {
}
MCSection *Sec = getContext().getMachOSection("__LLVM", "__cg_profile", 0,
SectionKind::getMetadata());
- llvm::copy(OS.str(), Sec->curFragList()->Head->getContents().data());
+ llvm::copy(OS.str(), Sec->curFragList()->Head->getVarContents().data());
}
unsigned NumSections = Asm.end() - Asm.begin();
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index d9680c7739a1b..7a8395a2e582b 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -1034,12 +1034,14 @@ MCELFStreamer &MipsTargetELFStreamer::getStreamer() {
void MipsTargetELFStreamer::emitGPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_GPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(8);
// fixup_Mips_GPREL32 desginates R_MIPS_GPREL32+R_MIPS_64 on MIPS64.
S.addFixup(Value, Mips::fixup_Mips_GPREL32);
S.appendContents(8, 0);
@@ -1047,24 +1049,28 @@ void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
void MipsTargetELFStreamer::emitDTPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_DTPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitDTPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(8);
S.addFixup(Value, Mips::fixup_Mips_DTPREL64);
S.appendContents(8, 0);
}
void MipsTargetELFStreamer::emitTPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_TPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitTPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(8);
S.addFixup(Value, Mips::fixup_Mips_TPREL64);
S.appendContents(8, 0);
}
More information about the llvm-commits
mailing list