[llvm] f36ce53 - MCFragment: Use trailing data for fixed-size part
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 28 22:12:18 PDT 2025
Author: Fangrui Song
Date: 2025-07-28T22:12:15-07:00
New Revision: f36ce53adf9294556c5d5f5f55c484c923f0c286
URL: https://github.com/llvm/llvm-project/commit/f36ce53adf9294556c5d5f5f55c484c923f0c286
DIFF: https://github.com/llvm/llvm-project/commit/f36ce53adf9294556c5d5f5f55c484c923f0c286.diff
LOG: MCFragment: Use trailing data for fixed-size part
The fixed-size content of the MCFragment object is now stored as
trailing data, replacing ContentStart/ContentEnd with ContentSize. The
available space for trailing data is tracked using `FragSpace`. If the
available space is insufficient, a new block is allocated within the
bump allocator `MCObjectStreamer::FragStorage`.
FragList::Tail cannot be reused when switching sections or subsections,
as it is not associated with the fragment space tracked by `FragSpace`.
Instead, allocate a new fragment, which becomes less expensive after #150574.
Data can only be appended to the tail fragment of a subsection, not to
fragments in the middle. Post-assembler-layout adjustments (such as
.llvm_addrsig and .llvm.call-graph-profile) have been updated to use the
variable-size part instead.
Pull Request: https://github.com/llvm/llvm-project/pull/150846
Added:
llvm/test/MC/ELF/many-instructions.s
Modified:
llvm/include/llvm/MC/MCObjectStreamer.h
llvm/include/llvm/MC/MCSection.h
llvm/lib/MC/MCObjectStreamer.cpp
llvm/lib/MC/MCWin64EH.cpp
llvm/lib/MC/MCWinCOFFStreamer.cpp
llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index 4b43a8fadb925..5ac7aba679ec4 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -52,6 +52,10 @@ class MCObjectStreamer : public MCStreamer {
DenseMap<const MCSymbol *, SmallVector<PendingAssignment, 1>>
pendingAssignments;
+ SmallVector<std::unique_ptr<char[]>, 0> FragStorage;
+ // Available bytes in the current block for trailing data or new fragments.
+ size_t FragSpace = 0;
+
void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &);
void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
@@ -84,11 +88,18 @@ class MCObjectStreamer : public MCStreamer {
// Add a fragment with a variable-size tail and start a new empty fragment.
void insert(MCFragment *F);
+ char *getCurFragEnd() const {
+ return reinterpret_cast<char *>(CurFrag + 1) + CurFrag->getFixedSize();
+ }
+ MCFragment *allocFragSpace(size_t Headroom);
// Add a new fragment to the current section without a variable-size tail.
void newFragment();
+ void ensureHeadroom(size_t Headroom);
void appendContents(ArrayRef<char> Contents);
void appendContents(size_t Num, char Elt);
+ // Add a fixup to the current fragment. Call ensureHeadroom beforehand to
+ // ensure the fixup and appended content apply to the same fragment.
void addFixup(const MCExpr *Value, MCFixupKind Kind);
void emitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override;
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 7989310e5a8f2..1579fa6646c3c 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -93,8 +93,7 @@ class MCFragment {
// Track content and fixups for the fixed-size part as fragments are
// appended to the section. The content remains immutable, except when
// modified by applyFixup.
- uint32_t ContentStart = 0;
- uint32_t ContentEnd = 0;
+ uint32_t FixedSize = 0;
uint32_t FixupStart = 0;
uint32_t FixupEnd = 0;
@@ -205,18 +204,6 @@ class MCFragment {
//== Content-related functions manage parent's storage using ContentStart and
// ContentSize.
- // Get a SmallVector reference. The caller should call doneAppending to update
- // `ContentEnd`.
- SmallVectorImpl<char> &getContentsForAppending();
- void doneAppending();
- void appendContents(ArrayRef<char> Contents) {
- getContentsForAppending().append(Contents.begin(), Contents.end());
- doneAppending();
- }
- void appendContents(size_t Num, char Elt) {
- getContentsForAppending().append(Num, Elt);
- doneAppending();
- }
MutableArrayRef<char> getContents();
ArrayRef<char> getContents() const;
@@ -225,10 +212,10 @@ class MCFragment {
MutableArrayRef<char> getVarContents();
ArrayRef<char> getVarContents() const;
- size_t getFixedSize() const { return ContentEnd - ContentStart; }
+ size_t getFixedSize() const { return FixedSize; }
size_t getVarSize() const { return VarContentEnd - VarContentStart; }
size_t getSize() const {
- return ContentEnd - ContentStart + (VarContentEnd - VarContentStart);
+ return FixedSize + (VarContentEnd - VarContentStart);
}
//== Fixup-related functions manage parent's storage using FixupStart and
@@ -651,28 +638,11 @@ class LLVM_ABI MCSection {
bool isBssSection() const { return IsBss; }
};
-inline SmallVectorImpl<char> &MCFragment::getContentsForAppending() {
- SmallVectorImpl<char> &S = getParent()->ContentStorage;
- if (LLVM_UNLIKELY(ContentEnd != S.size())) {
- // Move the elements to the end. Reserve space to avoid invalidating
- // S.begin()+I for `append`.
- auto Size = ContentEnd - ContentStart;
- auto I = std::exchange(ContentStart, S.size());
- S.reserve(S.size() + Size);
- S.append(S.begin() + I, S.begin() + I + Size);
- }
- return S;
-}
-inline void MCFragment::doneAppending() {
- ContentEnd = getParent()->ContentStorage.size();
-}
inline MutableArrayRef<char> MCFragment::getContents() {
- return MutableArrayRef(getParent()->ContentStorage)
- .slice(ContentStart, ContentEnd - ContentStart);
+ return {reinterpret_cast<char *>(this + 1), FixedSize};
}
inline ArrayRef<char> MCFragment::getContents() const {
- return ArrayRef(getParent()->ContentStorage)
- .slice(ContentStart, ContentEnd - ContentStart);
+ return {reinterpret_cast<const char *>(this + 1), FixedSize};
}
inline MutableArrayRef<char> MCFragment::getVarContents() {
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index e277143ac3899..17d096858d76d 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -46,27 +46,83 @@ MCAssembler *MCObjectStreamer::getAssemblerPtr() {
return nullptr;
}
+constexpr size_t FragBlockSize = 16384;
+// Ensure the new fragment can at least store a few bytes.
+constexpr size_t NewFragHeadroom = 8;
+
+static_assert(NewFragHeadroom >= alignof(MCFragment));
+static_assert(FragBlockSize >= sizeof(MCFragment) + NewFragHeadroom);
+
+MCFragment *MCObjectStreamer::allocFragSpace(size_t Headroom) {
+ auto Size = std::max(FragBlockSize, sizeof(MCFragment) + Headroom);
+ FragSpace = Size - sizeof(MCFragment);
+ auto Chunk = std::unique_ptr<char[]>(new char[Size]);
+ auto *F = reinterpret_cast<MCFragment *>(Chunk.get());
+ FragStorage.push_back(std::move(Chunk));
+ return F;
+}
+
void MCObjectStreamer::newFragment() {
- addFragment(getContext().allocFragment<MCFragment>());
+ MCFragment *F;
+ if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
+ auto End = reinterpret_cast<size_t>(getCurFragEnd());
+ F = reinterpret_cast<MCFragment *>(
+ alignToPowerOf2(End, alignof(MCFragment)));
+ FragSpace -= size_t(F) - End + sizeof(MCFragment);
+ } else {
+ F = allocFragSpace(0);
+ }
+ new (F) MCFragment();
+ addFragment(F);
+}
+
+void MCObjectStreamer::ensureHeadroom(size_t Headroom) {
+ if (Headroom <= FragSpace)
+ return;
+ auto *F = allocFragSpace(Headroom);
+ new (F) MCFragment();
+ addFragment(F);
}
-void MCObjectStreamer::insert(MCFragment *F) {
- assert(F->getKind() != MCFragment::FT_Data &&
+void MCObjectStreamer::insert(MCFragment *Frag) {
+ assert(Frag->getKind() != MCFragment::FT_Data &&
"F should have a variable-size tail");
+ // Frag is not connected to FragSpace. Before modifying CurFrag with
+ // addFragment(Frag), allocate an empty fragment to maintain FragSpace
+ // connectivity, potentially reusing CurFrag's associated space.
+ MCFragment *F;
+ if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
+ auto End = reinterpret_cast<size_t>(getCurFragEnd());
+ F = reinterpret_cast<MCFragment *>(
+ alignToPowerOf2(End, alignof(MCFragment)));
+ FragSpace -= size_t(F) - End + sizeof(MCFragment);
+ } else {
+ F = allocFragSpace(0);
+ }
+ new (F) MCFragment();
+
+ addFragment(Frag);
addFragment(F);
- newFragment();
}
void MCObjectStreamer::appendContents(ArrayRef<char> Contents) {
- CurFrag->appendContents(Contents);
+ ensureHeadroom(Contents.size());
+ assert(FragSpace >= Contents.size());
+ llvm::copy(Contents, getCurFragEnd());
+ CurFrag->FixedSize += Contents.size();
+ FragSpace -= Contents.size();
}
void MCObjectStreamer::appendContents(size_t Num, char Elt) {
- CurFrag->appendContents(Num, Elt);
+ ensureHeadroom(Num);
+ MutableArrayRef<char> Data(getCurFragEnd(), Num);
+ llvm::fill(Data, Elt);
+ CurFrag->FixedSize += Num;
+ FragSpace -= Num;
}
void MCObjectStreamer::addFixup(const MCExpr *Value, MCFixupKind Kind) {
- CurFrag->addFixup(MCFixup::create(CurFrag->getFixedSize(), Value, Kind));
+ CurFrag->addFixup(MCFixup::create(getCurFragSize(), Value, Kind));
}
// As a compile-time optimization, avoid allocating and evaluating an MCExpr
@@ -115,6 +171,8 @@ void MCObjectStreamer::reset() {
}
EmitEHFrame = true;
EmitDebugFrame = false;
+ FragStorage.clear();
+ FragSpace = 0;
MCStreamer::reset();
}
@@ -143,7 +201,6 @@ void MCObjectStreamer::emitCFISections(bool EH, bool Debug, bool SFrame) {
void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
SMLoc Loc) {
MCStreamer::emitValueImpl(Value, Size, Loc);
- MCFragment *DF = getCurrentFragment();
MCDwarfLineEntry::make(this, getCurrentSectionOnly());
@@ -158,9 +215,9 @@ void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
emitIntValue(AbsValue, Size);
return;
}
- DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
- MCFixup::getDataKindForSize(Size)));
- DF->appendContents(Size, 0);
+ ensureHeadroom(Size);
+ addFixup(Value, MCFixup::getDataKindForSize(Size));
+ appendContents(Size, 0);
}
MCSymbol *MCObjectStreamer::emitCFILabel() {
@@ -194,7 +251,7 @@ void MCObjectStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) {
// section.
MCFragment *F = CurFrag;
Symbol->setFragment(F);
- Symbol->setOffset(F->getContents().size());
+ Symbol->setOffset(F->getFixedSize());
emitPendingAssignments(Symbol);
}
@@ -260,6 +317,21 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
F0 = CurFrag;
}
+ // To maintain connectivity between CurFrag and FragSpace when CurFrag is
+ // modified, allocate an empty fragment and append it to the fragment list.
+ // (Subsections[I].second.Tail is not connected to FragSpace.)
+ MCFragment *F;
+ if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) {
+ auto End = reinterpret_cast<size_t>(getCurFragEnd());
+ F = reinterpret_cast<MCFragment *>(
+ alignToPowerOf2(End, alignof(MCFragment)));
+ FragSpace -= size_t(F) - End + sizeof(MCFragment);
+ } else {
+ F = allocFragSpace(0);
+ }
+ new (F) MCFragment();
+ F->setParent(Section);
+
auto &Subsections = Section->Subsections;
size_t I = 0, E = Subsections.size();
while (I != E && Subsections[I].first < Subsection)
@@ -267,13 +339,16 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
// If the subsection number is not in the sorted Subsections list, create a
// new fragment list.
if (I == E || Subsections[I].first != Subsection) {
- auto *F = getContext().allocFragment<MCFragment>();
- F->setParent(Section);
Subsections.insert(Subsections.begin() + I,
{Subsection, MCSection::FragList{F, F}});
+ Section->CurFragList = &Subsections[I].second;
+ CurFrag = F;
+ } else {
+ Section->CurFragList = &Subsections[I].second;
+ CurFrag = Subsections[I].second.Tail;
+ // Ensure CurFrag is associated with FragSpace.
+ addFragment(F);
}
- Section->CurFragList = &Subsections[I].second;
- CurFrag = Section->CurFragList->Tail;
// Define the section symbol at subsection 0's initial fragment if required.
if (!NewSec)
@@ -344,11 +419,15 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst,
MCFragment *F = getCurrentFragment();
// Append the instruction to the data fragment.
- size_t CodeOffset = F->getContents().size();
+ size_t CodeOffset = getCurFragSize();
+ SmallString<16> Content;
SmallVector<MCFixup, 1> Fixups;
- getAssembler().getEmitter().encodeInstruction(
- Inst, F->getContentsForAppending(), Fixups, STI);
- F->doneAppending();
+ getAssembler().getEmitter().encodeInstruction(Inst, Content, Fixups, STI);
+ appendContents(Content);
+ if (CurFrag != F) {
+ F = CurFrag;
+ CodeOffset = 0;
+ }
F->setHasInstructions(STI);
if (Fixups.empty())
@@ -356,19 +435,17 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst,
bool MarkedLinkerRelaxable = false;
for (auto &Fixup : Fixups) {
Fixup.setOffset(Fixup.getOffset() + CodeOffset);
- if (!Fixup.isLinkerRelaxable() || MarkedLinkerRelaxable)
+ if (!Fixup.isLinkerRelaxable())
continue;
- MarkedLinkerRelaxable = true;
- // Set the fragment's order within the subsection for use by
- // MCAssembler::relaxAlign.
- auto *Sec = F->getParent();
- if (!Sec->isLinkerRelaxable())
- Sec->setLinkerRelaxable();
+ F->setLinkerRelaxable();
// Do not add data after a linker-relaxable instruction. The
diff erence
// between a new label and a label at or before the linker-relaxable
// instruction cannot be resolved at assemble-time.
- F->setLinkerRelaxable();
- newFragment();
+ if (!MarkedLinkerRelaxable) {
+ MarkedLinkerRelaxable = true;
+ getCurrentSectionOnly()->setLinkerRelaxable();
+ newFragment();
+ }
}
F->appendFixups(Fixups);
}
diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp
index 72a8dd7031198..a87648afde7d6 100644
--- a/llvm/lib/MC/MCWin64EH.cpp
+++ b/llvm/lib/MC/MCWin64EH.cpp
@@ -318,6 +318,9 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
// Emit the epilog instructions.
if (EnableUnwindV2) {
+ // Ensure the fixups and appended content apply to the same fragment.
+ OS->ensureHeadroom(info->EpilogMap.size() * 2);
+
bool IsLast = true;
for (const auto &Epilog : llvm::reverse(info->EpilogMap)) {
if (IsLast) {
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index 1ffe25ccbc473..8be5054ad25e2 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -280,6 +280,7 @@ void MCWinCOFFStreamer::emitCOFFSymbolIndex(MCSymbol const *Symbol) {
void MCWinCOFFStreamer::emitCOFFSectionIndex(const MCSymbol *Symbol) {
visitUsedSymbol(*Symbol);
const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext());
+ ensureHeadroom(2);
addFixup(SRE, FK_SecRel_2);
appendContents(2, 0);
}
@@ -293,6 +294,7 @@ void MCWinCOFFStreamer::emitCOFFSecRel32(const MCSymbol *Symbol,
if (Offset)
MCE = MCBinaryExpr::createAdd(
MCE, MCConstantExpr::create(Offset, getContext()), getContext());
+ ensureHeadroom(4);
addFixup(MCE, FK_SecRel_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);
@@ -308,6 +310,7 @@ void MCWinCOFFStreamer::emitCOFFImgRel32(const MCSymbol *Symbol,
if (Offset)
MCE = MCBinaryExpr::createAdd(
MCE, MCConstantExpr::create(Offset, getContext()), getContext());
+ ensureHeadroom(4);
addFixup(MCE, FK_Data_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);
@@ -318,6 +321,7 @@ void MCWinCOFFStreamer::emitCOFFSecNumber(MCSymbol const *Symbol) {
// Create Symbol for section number.
const MCExpr *MCE = MCCOFFSectionNumberTargetExpr::create(
*Symbol, this->getWriter(), getContext());
+ ensureHeadroom(4);
addFixup(MCE, FK_Data_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);
@@ -328,6 +332,7 @@ void MCWinCOFFStreamer::emitCOFFSecOffset(MCSymbol const *Symbol) {
// Create Symbol for section offset.
const MCExpr *MCE =
MCCOFFSectionOffsetTargetExpr::create(*Symbol, getContext());
+ ensureHeadroom(4);
addFixup(MCE, FK_Data_4);
// Emit 4 bytes (zeros) to the object file.
appendContents(4, 0);
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index d9680c7739a1b..7a8395a2e582b 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -1034,12 +1034,14 @@ MCELFStreamer &MipsTargetELFStreamer::getStreamer() {
void MipsTargetELFStreamer::emitGPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_GPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(8);
// fixup_Mips_GPREL32 desginates R_MIPS_GPREL32+R_MIPS_64 on MIPS64.
S.addFixup(Value, Mips::fixup_Mips_GPREL32);
S.appendContents(8, 0);
@@ -1047,24 +1049,28 @@ void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
void MipsTargetELFStreamer::emitDTPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_DTPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitDTPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(8);
S.addFixup(Value, Mips::fixup_Mips_DTPREL64);
S.appendContents(8, 0);
}
void MipsTargetELFStreamer::emitTPRel32Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(4);
S.addFixup(Value, Mips::fixup_Mips_TPREL32);
S.appendContents(4, 0);
}
void MipsTargetELFStreamer::emitTPRel64Value(const MCExpr *Value) {
auto &S = getStreamer();
+ S.ensureHeadroom(8);
S.addFixup(Value, Mips::fixup_Mips_TPREL64);
S.appendContents(8, 0);
}
diff --git a/llvm/test/MC/ELF/many-instructions.s b/llvm/test/MC/ELF/many-instructions.s
new file mode 100644
index 0000000000000..cbdb2a71680d4
--- /dev/null
+++ b/llvm/test/MC/ELF/many-instructions.s
@@ -0,0 +1,10 @@
+# REQUIRES: asserts
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o /dev/null -debug-only=mc-dump
+
+## Test that encodeInstruction may cause a new fragment to be created.
+# CHECK: 0 Data Size:16200
+# CHECK: 16200 Data Size:180
+
+.rept 16384/10
+movabsq $foo, %rax
+.endr
More information about the llvm-commits
mailing list