[llvm] [X86] Reimplement bundle alignment mode (PR #175830)
Zachary Yedidia via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 2 15:16:27 PDT 2026
https://github.com/zyedidia updated https://github.com/llvm/llvm-project/pull/175830
>From 235684e6d434018f5a10edcac1b215b07dbb4345 Mon Sep 17 00:00:00 2001
From: Taehyun Noh <taehyun at utexas.edu>
Date: Mon, 12 Jan 2026 16:22:12 -0800
Subject: [PATCH 01/24] [X86] Reimplement bundle alignment mode
Reintroduce bundle alignment for X86 using a cleaner architecture that
builds on MCBoundaryAlignFragment infrastructure rather than per-fragment
padding fields. This preserves the MC simplifications from the original
removal while restoring functionality.
Key differences from the original implementation:
- Uses MCBoundaryAlignFragment instead of per-fragment BundlePadding
- Does not override emitInstToData (remains non-virtual)
- Adds prefix padding optimization (reuses --x86-pad-max-prefix-size)
- NOPs never span bundle boundaries
- X86-only
Co-Authored-By: Zachary Yedidia <zyedidia at gmail.com>
Co-Authored-By: Taehyun Noh <taehyun at utexas.edu>
---
llvm/include/llvm/MC/MCAssembler.h | 9 +
llvm/include/llvm/MC/MCELFStreamer.h | 8 +
llvm/include/llvm/MC/MCSection.h | 14 ++
llvm/include/llvm/MC/MCStreamer.h | 13 +
llvm/lib/MC/MCAsmStreamer.cpp | 21 ++
llvm/lib/MC/MCAssembler.cpp | 96 ++++++--
llvm/lib/MC/MCELFStreamer.cpp | 92 ++++++++
llvm/lib/MC/MCFragment.cpp | 4 +-
llvm/lib/MC/MCObjectStreamer.cpp | 16 ++
llvm/lib/MC/MCParser/AsmParser.cpp | 68 ++++++
llvm/lib/MC/MCStreamer.cpp | 3 +
.../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 222 ++++++++++++++++--
.../asm-printing-bundle-directives.s | 20 ++
.../X86/AlignedBundling/bundle-after-relax.s | 53 +++++
.../bundle-align-to-end-lock.s | 65 +++++
.../MC/X86/AlignedBundling/bundle-errors.s | 78 ++++++
.../test/MC/X86/AlignedBundling/bundle-inst.s | 36 +++
.../test/MC/X86/AlignedBundling/bundle-lock.s | 57 +++++
.../MC/X86/AlignedBundling/long-nop-pad.s | 40 ++++
.../MC/X86/AlignedBundling/prefix-padding.s | 69 ++++++
.../relax-for-prefix-padding.s | 48 ++++
llvm/test/MC/X86/AlignedBundling/section.s | 49 ++++
22 files changed, 1037 insertions(+), 44 deletions(-)
create mode 100644 llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-errors.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-inst.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-lock.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/prefix-padding.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/section.s
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 22f8ebde88756..2ede309321631 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -64,6 +64,8 @@ class MCAssembler {
// forward-reference displacements in `evaluateFixup`.
int64_t Stretch = 0;
+ unsigned BundleAlignSize = 0;
+
SectionListType Sections;
SmallVector<const MCSymbol *, 0> Symbols;
@@ -122,6 +124,9 @@ class MCAssembler {
void relaxDwarfCallFrameFragment(MCFragment &F);
void relaxSFrameFragment(MCFragment &DF);
+ /// Compute the padding size to boundary-align its connected fragments.
+ uint64_t computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF);
+
public:
/// Construct a new assembler instance.
//
@@ -198,6 +203,10 @@ class MCAssembler {
void setRelaxAll(bool Value) { RelaxAll = Value; }
int64_t getStretch() const { return Stretch; }
+ bool isBundlingEnabled() const { return BundleAlignSize != 0; }
+ unsigned getBundleAlignSize() const { return BundleAlignSize; }
+ void setBundleAlignSize(unsigned Size) { BundleAlignSize = Size; }
+
const_iterator begin() const { return Sections.begin(); }
const_iterator end() const { return Sections.end(); }
diff --git a/llvm/include/llvm/MC/MCELFStreamer.h b/llvm/include/llvm/MC/MCELFStreamer.h
index e1c86d8b12715..5d276d98afff9 100644
--- a/llvm/include/llvm/MC/MCELFStreamer.h
+++ b/llvm/include/llvm/MC/MCELFStreamer.h
@@ -69,6 +69,13 @@ class LLVM_ABI MCELFStreamer : public MCObjectStreamer {
void emitCGProfileEntry(const MCSymbolRefExpr *From,
const MCSymbolRefExpr *To, uint64_t Count) override;
+ void emitBundleAlignMode(Align Alignment) override;
+ void emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) override;
+ void emitBundleUnlock(const MCSubtargetInfo &STI) override;
+ bool isBundleLocked() const {
+ return getCurrentSectionOnly()->isBundleLocked();
+ }
+
// This is final. Override MCTargetStreamer::finish instead for
// target-specific code.
void finishImpl() final;
@@ -132,6 +139,7 @@ class LLVM_ABI MCELFStreamer : public MCObjectStreamer {
// GNU attributes that will get emitted at the end of the asm file.
SmallVector<AttributeItem, 64> GNUAttributes;
+ MCBoundaryAlignFragment *BundleBA = nullptr;
public:
void emitGNUAttribute(unsigned Tag, unsigned Value) override {
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 239a9654a4106..7a742de3837f6 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -545,6 +545,10 @@ class MCBoundaryAlignFragment : public MCFragment {
/// is not meaningful before that.
uint64_t Size = 0;
+ /// If true, align the last instruction in the fragment to the end of the
+ /// fragment.
+ bool AlignToEnd = false;
+
public:
MCBoundaryAlignFragment(Align AlignBoundary, const MCSubtargetInfo &STI)
: MCFragment(FT_BoundaryAlign), AlignBoundary(AlignBoundary) {
@@ -557,6 +561,9 @@ class MCBoundaryAlignFragment : public MCFragment {
Align getAlignment() const { return AlignBoundary; }
void setAlignment(Align Value) { AlignBoundary = Value; }
+ bool isAlignToEnd() const { return AlignToEnd; }
+ void setAlignToEnd(bool Value) { AlignToEnd = Value; }
+
const MCFragment *getLastFragment() const { return LastFragment; }
void setLastFragment(const MCFragment *F) {
assert(!F || getParent() == F->getParent());
@@ -610,6 +617,10 @@ class LLVM_ABI MCSection {
// fragment may not be fully resolved.
unsigned FirstLinkerRelaxable = -1u;
+ /// If bundle-locked, we ensure all instructions in the section are placed in
+ /// the same bundle.
+ bool IsBundleLocked = false;
+
/// Whether this section has had instructions emitted into it.
bool HasInstructions : 1;
@@ -676,6 +687,9 @@ class LLVM_ABI MCSection {
bool isLinkerRelaxable() const { return FirstLinkerRelaxable != -1u; }
void setFirstLinkerRelaxable(unsigned Order) { FirstLinkerRelaxable = Order; }
+ bool isBundleLocked() const { return IsBundleLocked; }
+ void setIsBundleLocked(bool Value) { IsBundleLocked = Value; }
+
MCFragment &getDummyFragment() { return DummyFragment; }
FragList *curFragList() const { return CurFragList; }
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index f1479f860e885..d7e212ffe7d70 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -1131,6 +1131,19 @@ class LLVM_ABI MCStreamer {
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym);
+ /// Set the bundle alignment mode from now on in the section.
+ /// The value 1 means turn the bundle alignment off.
+ virtual void emitBundleAlignMode(Align Alignment);
+
+ /// The following instructions are a bundle-locked group.
+ ///
+ /// \param AlignToEnd - If true, the bundle-locked group will be aligned to
+ /// the end of a bundle.
+ virtual void emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI);
+
+ /// Ends a bundle-locked group.
+ virtual void emitBundleUnlock(const MCSubtargetInfo &STI);
+
/// If this file is backed by a assembly streamer, this dumps the
/// specified string in the output .s file. This capability is indicated by
/// the hasRawTextSupport() predicate. By default this aborts.
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 037b44762875e..c7c061138c5bd 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -186,6 +186,9 @@ class MCAsmStreamer final : public MCAsmBaseStreamer {
raw_svector_ostream &OS) const;
void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
+ void emitBundleAlignMode(Align Alignment) override;
+ void emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) override;
+ void emitBundleUnlock(const MCSubtargetInfo &STI) override;
/// Helper to emit common .loc directive flags, isa, and discriminator.
void emitDwarfLocDirectiveFlags(unsigned Flags, unsigned Isa,
@@ -2648,6 +2651,24 @@ void MCAsmStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index,
EmitEOL();
}
+void MCAsmStreamer::emitBundleAlignMode(Align Alignment) {
+ OS << "\t.bundle_align_mode " << Log2(Alignment);
+ EmitEOL();
+}
+
+void MCAsmStreamer::emitBundleLock(bool AlignToEnd,
+ const MCSubtargetInfo &STI) {
+ OS << "\t.bundle_lock";
+ if (AlignToEnd)
+ OS << " align_to_end";
+ EmitEOL();
+}
+
+void MCAsmStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
+ OS << "\t.bundle_unlock";
+ EmitEOL();
+}
+
void MCAsmStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
const MCExpr *Expr, SMLoc) {
OS << "\t.reloc ";
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 85463fc2d56ed..16c32f1de2732 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -396,6 +396,35 @@ void MCAssembler::addRelocDirective(RelocDirective RD) {
relocDirectives.push_back(RD);
}
+/// Write NOPs while limiting the maximum NOP size.
+static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
+ uint64_t NumBytes, uint64_t FragmentOffset,
+ uint64_t MaxNopSize,
+ const MCSubtargetInfo *STI) {
+ uint64_t NumBytesEmitted = 0;
+ while (NumBytesEmitted < NumBytes) {
+ uint64_t NumBytesToEmit = std::min(NumBytes - NumBytesEmitted, MaxNopSize);
+
+ if (Asm.isBundlingEnabled()) {
+ unsigned BundleAlignSize = Asm.getBundleAlignSize();
+ uint64_t OffsetInBundle =
+ (FragmentOffset + NumBytesEmitted) & (BundleAlignSize - 1);
+ uint64_t SpaceInBundle = BundleAlignSize - OffsetInBundle;
+ NumBytesToEmit = std::min(NumBytesToEmit, SpaceInBundle);
+ }
+
+ assert(NumBytesToEmit && "try to emit zero-sized NOP");
+
+ if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit, STI)) {
+ report_fatal_error("unable to write NOP sequence of the remaining " +
+ Twine(NumBytesToEmit) + " bytes");
+ return;
+ }
+
+ NumBytesEmitted += NumBytesToEmit;
+ }
+}
+
/// Write the fragment \p F to the output file.
static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
const MCFragment &F) {
@@ -544,26 +573,22 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
if (!ControlledNopLength)
ControlledNopLength = MaximumNopLength;
- while (NumBytes) {
- uint64_t NumBytesToEmit =
- (uint64_t)std::min(NumBytes, ControlledNopLength);
- assert(NumBytesToEmit && "try to emit empty NOP instruction");
- if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit,
- NF.getSubtargetInfo())) {
- report_fatal_error("unable to write nop sequence of the remaining " +
- Twine(NumBytesToEmit) + " bytes");
- break;
- }
- NumBytes -= NumBytesToEmit;
- }
+ writeControlledNops(OS, Asm, (uint64_t)NumBytes, Asm.getFragmentOffset(NF),
+ (uint64_t)ControlledNopLength, NF.getSubtargetInfo());
break;
}
case MCFragment::FT_BoundaryAlign: {
const MCBoundaryAlignFragment &BF = cast<MCBoundaryAlignFragment>(F);
- if (!Asm.getBackend().writeNopData(OS, FragmentSize, BF.getSubtargetInfo()))
- report_fatal_error("unable to write nop sequence of " +
- Twine(FragmentSize) + " bytes");
+ if (!Asm.isBundlingEnabled()) {
+ if (!Asm.getBackend().writeNopData(OS, FragmentSize,
+ BF.getSubtargetInfo()))
+ report_fatal_error("unable to write nop sequence of " +
+ Twine(FragmentSize) + " bytes");
+ } else {
+ writeControlledNops(OS, Asm, FragmentSize, Asm.getFragmentOffset(BF),
+ FragmentSize, BF.getSubtargetInfo());
+ }
break;
}
@@ -969,11 +994,10 @@ static bool needPadding(uint64_t StartAddr, uint64_t Size,
isAgainstBoundary(StartAddr, Size, BoundaryAlignment);
}
-void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
- // BoundaryAlignFragment that doesn't need to align any fragment should not be
- // relaxed.
+uint64_t
+MCAssembler::computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) {
if (!BF.getLastFragment())
- return;
+ return 0;
uint64_t AlignedOffset = getFragmentOffset(BF);
uint64_t AlignedSize = 0;
@@ -984,9 +1008,32 @@ void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
}
Align BoundaryAlignment = BF.getAlignment();
- uint64_t NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
- ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
- : 0U;
+
+ uint64_t NewSize = 0;
+ if (isBundlingEnabled()) {
+ // For bundle alignment, we only pad instructions that cross the boundary.
+ NewSize = mayCrossBoundary(AlignedOffset, AlignedSize, BoundaryAlignment)
+ ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
+ : 0U;
+ if (BF.isAlignToEnd()) {
+ NewSize =
+ offsetToAlignment(AlignedOffset + AlignedSize, BoundaryAlignment);
+ }
+ } else {
+ NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
+ ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
+ : 0U;
+ }
+ return NewSize;
+}
+
+void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
+ // BoundaryAlignFragment that doesn't need to align any fragment should not be
+ // relaxed.
+ if (!BF.getLastFragment())
+ return;
+
+ uint64_t NewSize = computeBoundaryAlignSize(BF);
if (NewSize == BF.getSize())
return;
BF.setSize(NewSize);
@@ -1054,7 +1101,10 @@ void MCAssembler::relaxFragment(MCFragment &F) {
relaxAlign(F);
break;
case MCFragment::FT_Relaxable:
- assert(!getRelaxAll() && "Did not expect a FT_Relaxable in RelaxAll mode");
+ // Bundling emits every instruction as relaxable, so FT_Relaxable is
+ // expected with RelaxAll mode once bundling is enabled.
+ assert((isBundlingEnabled() || !getRelaxAll()) &&
+ "Did not expect a FT_Relaxable in RelaxAll mode");
relaxInstruction(F);
break;
case MCFragment::FT_LEB:
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index a18b27b2dc132..e943711fb9de9 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -80,6 +80,22 @@ void MCELFStreamer::emitLabelAtPos(MCSymbol *S, SMLoc Loc, MCFragment &F,
void MCELFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
MCAssembler &Asm = getAssembler();
+ MCFragment *CF = getCurrentFragment();
+ if (Asm.isBundlingEnabled()) {
+ if (isBundleLocked()) {
+ getContext().reportError(
+ getStartTokLoc(),
+ "unterminated .bundle_lock when changing a section");
+ // Clean up bundle state to allow continuing.
+ MCSection *CurSec = CF->getParent();
+ CurSec->setIsBundleLocked(false);
+ BundleBA = nullptr;
+ }
+
+ // Ensure the previous section gets aligned if necessary.
+ if (Asm.isBundlingEnabled() && CF->getParent()->hasInstructions())
+ Section->ensureMinAlignment(Align(Asm.getBundleAlignSize()));
+ }
auto *SectionELF = static_cast<const MCSectionELF *>(Section);
const MCSymbol *Grp = SectionELF->getGroup();
if (Grp)
@@ -316,6 +332,82 @@ void MCELFStreamer::emitIdent(StringRef IdentString) {
popSection();
}
+void MCELFStreamer::emitBundleAlignMode(Align Alignment) {
+ if (Log2(Alignment) > 30)
+ getContext().reportError(getStartTokLoc(),
+ ".bundle_align_mode alignment must be <= 30");
+ MCAssembler &Assembler = getAssembler();
+ setAllowAutoPadding(true);
+
+ if (Alignment > 1 && (Assembler.getBundleAlignSize() == 0 ||
+ Assembler.getBundleAlignSize() == Alignment.value()))
+ Assembler.setBundleAlignSize(Alignment.value());
+ else
+ getContext().reportError(getStartTokLoc(),
+ ".bundle_align_mode cannot be changed once set");
+}
+
+void MCELFStreamer::emitBundleLock(bool AlignToEnd,
+ const MCSubtargetInfo &STI) {
+ MCSection &Sec = *getCurrentSectionOnly();
+ auto &Asm = getAssembler();
+
+ if (!Asm.isBundlingEnabled()) {
+ getContext().reportError(
+ getStartTokLoc(), ".bundle_lock forbidden when bundling is disabled");
+ return;
+ }
+
+ if (Sec.isBundleLocked()) {
+ getContext().reportError(getStartTokLoc(),
+ "nested .bundle_lock is not allowed");
+ return;
+ }
+ Sec.setIsBundleLocked(true);
+
+ auto AlignBoundary = Asm.getBundleAlignSize();
+ BundleBA =
+ newSpecialFragment<MCBoundaryAlignFragment>(Align(AlignBoundary), STI);
+ BundleBA->setAlignToEnd(AlignToEnd);
+}
+
+void MCELFStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
+ MCSection &Sec = *getCurrentSectionOnly();
+
+ if (!getAssembler().isBundlingEnabled()) {
+ getContext().reportError(
+ getStartTokLoc(), ".bundle_unlock forbidden when bundling is disabled");
+ return;
+ }
+ if (!isBundleLocked()) {
+ getContext().reportError(getStartTokLoc(),
+ ".bundle_unlock without matching lock");
+ return;
+ }
+
+ Sec.setIsBundleLocked(false);
+
+ MCFragment *CF = getCurrentFragment();
+ BundleBA->setLastFragment(CF);
+ // Bundle overflow check.
+ uint64_t AlignedSize = 0;
+ for (const MCFragment *F = BundleBA->getNext();; F = F->getNext()) {
+ AlignedSize += getAssembler().computeFragmentSize(*F);
+ if (F == BundleBA->getLastFragment())
+ break;
+ }
+ BundleBA = nullptr;
+
+ if (AlignedSize > getAssembler().getBundleAlignSize())
+ getContext().reportError(getStartTokLoc(),
+ "fragment can't be larger than a bundle size");
+
+ newFragment();
+
+ CF->getParent()->ensureMinAlignment(
+ Align(getAssembler().getBundleAlignSize()));
+}
+
void MCELFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *Sym,
uint64_t Offset,
const MCSymbolRefExpr *&SRE) {
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index 21a304da0bb4f..78c955e4f3d1a 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -167,8 +167,8 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
case MCFragment::FT_BoundaryAlign: {
const auto *BF = cast<MCBoundaryAlignFragment>(this);
OS << " BoundarySize:" << BF->getAlignment().value()
- << " LastFragment:" << BF->getLastFragment()
- << " Size:" << BF->getSize();
+ << " LastFragment:" << BF->getLastFragment() << " Size:" << BF->getSize()
+ << " AlignToEnd:" << BF->isAlignToEnd();
break;
}
case MCFragment::FT_PrefAlign:
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index d89e5df706b21..bc98902df6faf 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -415,6 +415,22 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
// If this instruction doesn't need relaxation, just emit it as data.
MCAssembler &Assembler = getAssembler();
MCAsmBackend &Backend = Assembler.getBackend();
+
+ // To enable better bundle-nop optimization, we emit every instruction
+ // as a relaxable fragment.
+ if (Assembler.isBundlingEnabled()) {
+ if (Sec->isBundleLocked() || Assembler.getRelaxAll()) {
+ MCInst Relaxed = Inst;
+ while (Backend.mayNeedRelaxation(Relaxed.getOpcode(),
+ Relaxed.getOperands(), STI))
+ Backend.relaxInstruction(Relaxed, STI);
+ emitInstToFragment(Relaxed, STI);
+ } else {
+ emitInstToFragment(Inst, STI);
+ }
+ return;
+ }
+
if (!(Backend.mayNeedRelaxation(Inst.getOpcode(), Inst.getOperands(), STI) ||
Backend.allowEnhancedRelaxation())) {
emitInstToData(Inst, STI);
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index e0ad48f6e1932..4bc7be913c2b2 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -422,6 +422,9 @@ class AsmParser : public MCAsmParser {
DK_ORG,
DK_FILL,
DK_ENDR,
+ DK_BUNDLE_ALIGN_MODE,
+ DK_BUNDLE_LOCK,
+ DK_BUNDLE_UNLOCK,
DK_ZERO,
DK_EXTERN,
DK_GLOBL,
@@ -711,6 +714,13 @@ class AsmParser : public MCAsmParser {
bool parseDirectiveAddrsig();
bool parseDirectiveAddrsigSym();
+ // ".bundle_align_mode"
+ bool parseDirectiveBundleAlignMode();
+ // ".bundle_lock"
+ bool parseDirectiveBundleLock();
+ // ".bundle_unlock"
+ bool parseDirectiveBundleUnlock();
+
void initializeDirectiveKindMap();
void initializeCVDefRangeTypeMap();
};
@@ -2076,6 +2086,12 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
return parseDirectiveIrpc(IDLoc);
case DK_ENDR:
return parseDirectiveEndr(IDLoc);
+ case DK_BUNDLE_ALIGN_MODE:
+ return parseDirectiveBundleAlignMode();
+ case DK_BUNDLE_LOCK:
+ return parseDirectiveBundleLock();
+ case DK_BUNDLE_UNLOCK:
+ return parseDirectiveBundleUnlock();
case DK_SLEB128:
return parseDirectiveLEB128(true);
case DK_ULEB128:
@@ -5629,6 +5645,9 @@ void AsmParser::initializeDirectiveKindMap() {
DirectiveKindMap[".irp"] = DK_IRP;
DirectiveKindMap[".irpc"] = DK_IRPC;
DirectiveKindMap[".endr"] = DK_ENDR;
+ DirectiveKindMap[".bundle_align_mode"] = DK_BUNDLE_ALIGN_MODE;
+ DirectiveKindMap[".bundle_lock"] = DK_BUNDLE_LOCK;
+ DirectiveKindMap[".bundle_unlock"] = DK_BUNDLE_UNLOCK;
DirectiveKindMap[".if"] = DK_IF;
DirectiveKindMap[".ifeq"] = DK_IFEQ;
DirectiveKindMap[".ifge"] = DK_IFGE;
@@ -5988,6 +6007,55 @@ bool AsmParser::parseDirectiveAddrsigSym() {
return false;
}
+/// parseDirectiveBundleAlignMode
+/// ::= {.bundle_align_mode} expression
+bool AsmParser::parseDirectiveBundleAlignMode() {
+ // Expect a single argument: an expression that evaluates to a constant
+ // in the inclusive range 0-30.
+ SMLoc ExprLoc = getLexer().getLoc();
+ int64_t AlignSizePow2;
+ if (checkForValidSection() || parseAbsoluteExpression(AlignSizePow2) ||
+ parseEOL() ||
+ check(AlignSizePow2 < 0 || AlignSizePow2 > 30, ExprLoc,
+ "invalid bundle alignment size (expected between 0 and 30)"))
+ return true;
+
+ getStreamer().emitBundleAlignMode(Align(1ULL << AlignSizePow2));
+ return false;
+}
+
+/// parseDirectiveBundleLock
+/// ::= {.bundle_lock} [align_to_end]
+bool AsmParser::parseDirectiveBundleLock() {
+ if (checkForValidSection())
+ return true;
+ bool AlignToEnd = false;
+
+ StringRef Option;
+ SMLoc Loc = getTok().getLoc();
+ const char *InvalidOptionError = "invalid option for `.bundle_lock`";
+
+ if (!parseOptionalToken(AsmToken::EndOfStatement)) {
+ if (check(parseIdentifier(Option), Loc, InvalidOptionError) ||
+ check(Option != "align_to_end", Loc, InvalidOptionError) || parseEOL())
+ return true;
+ AlignToEnd = true;
+ }
+
+ getStreamer().emitBundleLock(AlignToEnd, getTargetParser().getSTI());
+ return false;
+}
+
+/// parseDirectiveBundleUnlock
+/// ::= {.bundle_unlock}
+bool AsmParser::parseDirectiveBundleUnlock() {
+ if (checkForValidSection() || parseEOL())
+ return true;
+
+ getStreamer().emitBundleUnlock(getTargetParser().getSTI());
+ return false;
+}
+
bool AsmParser::parseDirectivePseudoProbe() {
int64_t Guid;
int64_t Index;
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 1cd94715b2592..e4fc9d4932ee6 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -1527,6 +1527,9 @@ void MCStreamer::emitCodeAlignment(Align Alignment, const MCSubtargetInfo &STI,
unsigned MaxBytesToEmit) {}
void MCStreamer::emitValueToOffset(const MCExpr *Offset, unsigned char Value,
SMLoc Loc) {}
+void MCStreamer::emitBundleAlignMode(Align Alignment) {}
+void MCStreamer::emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) {}
+void MCStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {}
void MCStreamer::finishImpl() {}
bool MCStreamer::popSection() {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 0b3da6582d907..72affa7289ce3 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -134,6 +134,8 @@ class X86AsmBackend : public MCAsmBackend {
bool needAlign(const MCInst &Inst) const;
bool canPadBranches(MCObjectStreamer &OS) const;
bool canPadInst(const MCInst &Inst, MCObjectStreamer &OS) const;
+ void emitInstructionBeginBundle(MCObjectStreamer &OS);
+ void emitInstructionEndBundle(MCObjectStreamer &OS);
public:
X86AsmBackend(const Target &T, const MCSubtargetInfo &STI)
@@ -198,6 +200,12 @@ class X86AsmBackend : public MCAsmBackend {
bool finishLayout() const override;
+ bool padInstsBackward(SmallVectorImpl<MCFragment *> &Relaxable,
+ unsigned &RemainingSize) const;
+ bool dividePadInBundle(const MCAssembler &Asm,
+ ArrayRef<MCFragment *> Peephole) const;
+ bool optimizeBundleNops(const MCAssembler &Asm) const;
+
unsigned getMaximumNopSize(const MCSubtargetInfo &STI) const override;
bool writeNopData(raw_ostream &OS, uint64_t Count,
@@ -464,9 +472,55 @@ void X86_MC::emitInstruction(MCObjectStreamer &S, const MCInst &Inst,
Backend.emitInstructionEnd(S, Inst);
}
+/// If the upcoming instruction is inside the bundle lock, do nothing so that
+/// the ObjectStreamer emits the instruction to the current fragment. If not, it
+/// creates a new BA to group bundled fragments.
+void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
+ assert(Asm->isBundlingEnabled());
+
+ if (OS.getCurrentSectionOnly()->isBundleLocked()) {
+ OS.getCurrentFragment()->setAllowAutoPadding(true);
+ return;
+ }
+ PendingBA = OS.newSpecialFragment<MCBoundaryAlignFragment>(
+ Align(Asm->getBundleAlignSize()), STI);
+ // We can set LastFragment now, before the instruction is emitted, as bundling
+ // emits one fragment per instruction. Deferring setLastFragment to
+ // post-emitInstruction would risk capturing a fragment that a subsequent
+ // emitCodeAlignment repurposes in-place to FT_Align, corrupting the BA's
+ // boundary range.
+ PendingBA->setLastFragment(OS.getCurrentFragment());
+
+ OS.getCurrentFragment()->setAllowAutoPadding(true);
+}
+
+/// If the just-emitted instruction is inside the bundle lock, check the current
+/// fragment is non-zero to ensure the instruction is placed as expected. If it
+/// is not locked, finalize pending BA Fragment. emitBundleUnlock will close the
+/// fragment and start a new empty fragment.
+void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
+ assert(Asm->isBundlingEnabled());
+
+ MCFragment *CF = OS.getCurrentFragment();
+
+ if (OS.getCurrentSectionOnly()->isBundleLocked()) {
+ // We're still inside the lock, do not close the current fragment with BA.
+ return;
+ }
+ assert(PendingBA && "MCBoundaryAlignFragment is expected for every "
+ "instruction if it is not bundle-locked");
+
+ PendingBA = nullptr;
+
+ CF->getParent()->ensureMinAlignment(Align(Asm->getBundleAlignSize()));
+}
+
/// Insert BoundaryAlignFragment before instructions to align branches.
void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
- const MCInst &Inst, const MCSubtargetInfo &STI) {
+ const MCInst &Inst,
+ const MCSubtargetInfo &STI) {
+ if (Asm->isBundlingEnabled())
+ return emitInstructionBeginBundle(OS);
bool CanPadInst = canPadInst(Inst, OS);
if (CanPadInst)
OS.getCurrentFragment()->setAllowAutoPadding(true);
@@ -529,6 +583,8 @@ void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
/// Set the last fragment to be aligned for the BoundaryAlignFragment.
void X86AsmBackend::emitInstructionEnd(MCObjectStreamer &OS,
const MCInst &Inst) {
+ if (Asm->isBundlingEnabled())
+ return emitInstructionEndBundle(OS);
// Update PrevInstOpcode here, canPadInst() reads that.
MCFragment *CF = OS.getCurrentFragment();
PrevInstOpcode = Inst.getOpcode();
@@ -736,6 +792,14 @@ bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFragment &,
const MCValue &Target,
uint64_t Value,
bool Resolved) const {
+ if (Asm->isBundlingEnabled() && Resolved) {
+ // This ensures remaining short branches have sufficient headroom to survive
+ // any intra-bundle shift caused by prefix padding in dividePadInBundle.
+ auto BundleAlignSize = Asm->getBundleAlignSize();
+ return (!isInt<8>(Value + BundleAlignSize) ||
+ !isInt<8>(Value - BundleAlignSize)) ||
+ Target.getSpecifier();
+ }
// If resolved, relax if the value is too big for a (signed) i8.
//
// Currently, `jmp local at plt` relaxes JMP even if the offset is small,
@@ -853,7 +917,144 @@ bool X86AsmBackend::padInstructionEncoding(MCFragment &RF,
return Changed;
}
+bool X86AsmBackend::padInstsBackward(SmallVectorImpl<MCFragment *> &Relaxable,
+ unsigned &RemainingSize) const {
+ bool Changed = false;
+ while (!Relaxable.empty() && RemainingSize != 0) {
+ auto &RF = *Relaxable.pop_back_val();
+ // Give the backend a chance to play any tricks it wishes to increase
+ // the encoding size of the given instruction. Target independent code
+ // will try further relaxation, but target's may play further tricks.
+ Changed |= padInstructionEncoding(RF, Asm->getEmitter(), RemainingSize);
+
+ // If we have an instruction which hasn't been fully relaxed, we can't
+ // skip past it and insert bytes before it. Changing its starting
+ // offset might require a larger negative offset than it can encode.
+ // We don't need to worry about larger positive offsets as none of the
+ // possible offsets between this and our align are visible, and the
+ // ones afterwards aren't changing.
+ if (mayNeedRelaxation(RF.getOpcode(), RF.getOperands(),
+ *RF.getSubtargetInfo()))
+ break;
+ }
+ Relaxable.clear();
+ return Changed;
+}
+
+// Peephole is a list of Fragments that ends with non-zero-sized
+// BoundaryAlignFragment. Most of the time it will be every instruction within a
+// bundle, but there can be a partial bundle if it has nops in the middle(e.g.,
+// align_to_end).
+bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
+ ArrayRef<MCFragment *> Peephole) const {
+ bool Changed = false;
+
+ // Last Fragment is either FT_Align or FT_BoundaryAlign
+ auto *LastF = Peephole.back();
+ unsigned RemainingSize =
+ Asm.computeFragmentSize(*LastF) - LastF->getFixedSize();
+
+ unsigned StartOffset = Asm.getFragmentOffset(*LastF);
+ unsigned EndOffset = StartOffset + RemainingSize;
+ auto BoundaryAlignment = Align(Asm.getBundleAlignSize());
+ bool CrossBoundary = (StartOffset >> Log2(BoundaryAlignment)) !=
+ ((EndOffset - 1) >> Log2(BoundaryAlignment));
+
+ if (CrossBoundary) {
+ // i.e., this pad is a mix of suffix fragment of one bundle + prefix of the
+ // very next bundle. It prevents overflow of the first bundle when Peephole
+ // contains more than one bundle.
+ //
+ // This design limits the possibly further-optimized code, which might be
+ // achieved by migrating some instructions to the next bundle, but doing
+ // such may cause fixup errors because instructions can shift by more than
+ // a bundle-size and labels may become unreachable. Until we come up with a
+ // better logic, we limits the optimization scope to a single bundle.
+ RemainingSize -= EndOffset % Asm.getBundleAlignSize();
+ }
+ assert(RemainingSize > 0);
+
+ SmallVector<MCFragment *, 4> Relaxable;
+ for (auto *FIB : Peephole) {
+ if (FIB->getKind() == MCFragment::FT_Data) // Skip and ignore
+ continue;
+
+ if (FIB->getKind() == MCFragment::FT_Align) {
+ // p2align within a bundle
+ Relaxable.clear();
+ continue;
+ }
+
+ if (FIB->getKind() == MCFragment::FT_Relaxable) {
+ auto &RF = cast<MCFragment>(*FIB);
+ Relaxable.push_back(&RF);
+ continue;
+ }
+ }
+
+ // First, try padding previous instructions.
+ Changed |= padInstsBackward(Relaxable, RemainingSize);
+
+ // Second, try padding following instructions.
+ auto padInstsForward = [&](unsigned &Size) {
+ auto *BF = cast<MCBoundaryAlignFragment>(LastF);
+ for (auto *F = BF->getNext();; F = F->getNext()) {
+ if (F->getKind() == MCFragment::FT_Relaxable)
+ Changed |= padInstructionEncoding(*F, Asm.getEmitter(), Size);
+ if (F == BF->getLastFragment() || Size == 0)
+ break;
+ }
+ };
+
+ unsigned TailSize = EndOffset % Asm.getBundleAlignSize();
+ if (!CrossBoundary && RemainingSize > 0 && TailSize != 0) {
+ padInstsForward(RemainingSize);
+ } else if (CrossBoundary && TailSize > 0) {
+ unsigned NextRemainingSize = TailSize;
+ padInstsForward(NextRemainingSize);
+ RemainingSize += NextRemainingSize;
+ }
+
+ // FT_Align sizes will be recalculated by layoutSection(),
+ // FT_BoundaryAlign sizes are adjusted here.
+ if (auto *BF = dyn_cast<MCBoundaryAlignFragment>(LastF))
+ BF->setSize(RemainingSize);
+
+ return Changed;
+}
+
+bool X86AsmBackend::optimizeBundleNops(const MCAssembler &Asm) const {
+ bool Changed = false;
+ for (MCSection &Sec : Asm) {
+ if (!Sec.isText())
+ continue;
+
+ SmallVector<MCFragment *, 4> Bundle;
+ for (MCSection::iterator I = Sec.begin(), IE = Sec.end(); I != IE; ++I) {
+ MCFragment &F = *I;
+
+ if (F.getKind() == llvm::MCFragment::FT_BoundaryAlign) {
+ unsigned RemainingSize = Asm.computeFragmentSize(F) - F.getFixedSize();
+ if (RemainingSize > 0) {
+ Bundle.push_back(&F);
+ Changed |= dividePadInBundle(Asm, Bundle);
+ Bundle.clear();
+ continue;
+ }
+ }
+
+ if (Asm.getFragmentOffset(F) % Asm.getBundleAlignSize() == 0)
+ Bundle.clear(); // start a new bundle
+ Bundle.push_back(&F);
+ }
+ }
+
+ return Changed;
+}
+
bool X86AsmBackend::finishLayout() const {
+ if (Asm->isBundlingEnabled() && TargetPrefixMax != 0)
+ return optimizeBundleNops(*Asm);
// See if we can further relax some instructions to cut down on the number of
// nop bytes required for code alignment. The actual win is in reducing
// instruction count, not number of bytes. Modern X86-64 can easily end up
@@ -912,24 +1113,7 @@ bool X86AsmBackend::finishLayout() const {
// of the resulting code. If we later find a reason to expand
// particular instructions over others, we can adjust.
unsigned RemainingSize = Asm->computeFragmentSize(F) - F.getFixedSize();
- while (!Relaxable.empty() && RemainingSize != 0) {
- auto &RF = *Relaxable.pop_back_val();
- // Give the backend a chance to play any tricks it wishes to increase
- // the encoding size of the given instruction. Target independent code
- // will try further relaxation, but target's may play further tricks.
- Changed |= padInstructionEncoding(RF, Asm->getEmitter(), RemainingSize);
-
- // If we have an instruction which hasn't been fully relaxed, we can't
- // skip past it and insert bytes before it. Changing its starting
- // offset might require a larger negative offset than it can encode.
- // We don't need to worry about larger positive offsets as none of the
- // possible offsets between this and our align are visible, and the
- // ones afterwards aren't changing.
- if (mayNeedRelaxation(RF.getOpcode(), RF.getOperands(),
- *RF.getSubtargetInfo()))
- break;
- }
- Relaxable.clear();
+ padInstsBackward(Relaxable, RemainingSize);
// If we're looking at a boundary align, make sure we don't try to pad
// its target instructions for some following directive. Doing so would
diff --git a/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s b/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
new file mode 100644
index 0000000000000..fcd53711e656e
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
@@ -0,0 +1,20 @@
+# RUN: llvm-mc -filetype=asm -triple x86_64 %s -o - 2>&1 | FileCheck %s
+
+## Just a simple test for the assembly emitter - making sure it emits back the
+## bundling directives.
+
+ .text
+foo:
+ .bundle_align_mode 4
+# CHECK: .bundle_align_mode 4
+ pushq %rbp
+ .bundle_lock
+# CHECK: .bundle_lock
+ cmpl %r14d, %ebp
+ jle .L_ELSE
+ .bundle_unlock
+# CHECK: .bundle_unlock
+ .bundle_lock align_to_end
+# CHECK: .bundle_lock align_to_end
+ add %rbx, %rdx
+ .bundle_unlock
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s b/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
new file mode 100644
index 0000000000000..41803b6f8d38c
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
@@ -0,0 +1,53 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d - | FileCheck %s
+
+## Test that instructions inside bundle-locked groups are relaxed even if their
+## fixup is short enough not to warrant relaxation on its own.
+ .text
+relax_in_bundle:
+ .bundle_align_mode 4
+ pushq %rbp
+
+ movl %edi, %ebx
+ callq bar
+ movl %eax, %r14d
+ imull $17, %ebx, %ebp
+ movl %ebx, %edi
+ callq bar
+ cmpl %r14d, %ebp
+ .bundle_lock
+
+ jle .L_ELSE
+## This group would've started at 0x18 and is too long, so a chunky NOP padding
+## is inserted to push it to 0x20.
+# CHECK: 18: {{[a-f0-9 ]+}} nopl
+
+## The long encoding for JLE should be used here even though its target is close
+# CHECK-NEXT: 20: 0f 8e
+
+ addl %ebp, %eax
+
+ jmp .L_RET
+## Same for the JMP
+# CHECK: 28: e9
+
+ .bundle_unlock
+
+.L_ELSE:
+ imull %ebx, %eax
+.L_RET:
+
+ popq %rbx
+
+## Test that an instruction near a bundle end gets properly padded to the next
+## bundle after it is relaxed.
+ .align 16
+relax_at_bundle_end:
+ .rept 14
+ push %rax
+ .endr
+# CHECK: 4d: 50 pushq
+# CHECK-NEXT: 4e: {{[a-f0-9 ]+}} nop
+# CHECK-NEXT: 50: 0f 85
+ jne 0x100
+
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s b/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
new file mode 100644
index 0000000000000..ef3dddc2a4ce2
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
@@ -0,0 +1,65 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test NOP padding for `align_to_end` bundle-locked groups.
+
+ .text
+foo:
+ .bundle_align_mode 4
+
+## Each of these callq instructions is 5 bytes long
+ callq bar
+ callq bar
+ .bundle_lock align_to_end
+ callq bar
+ .bundle_unlock
+## To align this group to a bundle end, we need a 1-byte NOP.
+# CHECK: a: nop
+# CHECK-NEXT: b: callq
+
+ callq bar
+ callq bar
+ .bundle_lock align_to_end
+ callq bar
+ callq bar
+ .bundle_unlock
+## Here we have to pad until the end of the *next* boundary because
+## otherwise the group crosses a boundary.
+# CHECK: 1a: nop
+## The nop sequence may be implemented as one instruction or many, but if
+## it's one instruction, that instruction cannot itself cross the boundary.
+# CHECK: 20: nop
+# CHECK-NEXT: 26: callq
+# CHECK-NEXT: 2b: callq
+
+ .align 16, 0x90
+lock_to_next_bundle:
+ .rept 14
+ inc %eax
+ .endr
+ .bundle_lock align_to_end
+ inc %eax
+ inc %eax
+ inc %eax
+ .bundle_unlock
+## This bundle group must be adjusted to the next boundary,
+## but nop optimization must not break the bundle too.
+# CHECK: 30: inc
+# CHECK: 4a: inc
+# CHECK-NEXT: 4c: nop
+# CHECK-NEXT: 50: nop
+# CHECK-NEXT: 5a: inc
+
+lock_fit_exactly:
+ .rept 14
+ inc %eax
+ .endr
+ .bundle_lock align_to_end
+ inc %eax
+ inc %eax
+ .bundle_unlock
+# CHECK: 60: inc
+# CHECK: 7e: inc
+
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
new file mode 100644
index 0000000000000..70e9795b74363
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
@@ -0,0 +1,78 @@
+# RUN: split-file %s %t
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/lock-without-mode.s 2>&1 | FileCheck %t/lock-without-mode.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/mode-without-arg.s 2>&1 | FileCheck %t/mode-without-arg.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/unlock-without-lock.s 2>&1 | FileCheck %t/unlock-without-lock.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/bad-lock-option.s 2>&1 | FileCheck %t/bad-lock-option.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/switch-section-locked.s 2>&1 | FileCheck %t/switch-section-locked.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/nested-lock.s 2>&1 | FileCheck %t/nested-lock.s
+
+
+## .bundle_lock can't come without a .bundle_align_mode before it
+#--- lock-without-mode.s
+ imull $17, %ebx, %ebp
+# CHECK: [[#@LINE+1]]:3: error: .bundle_lock forbidden when bundling is disabled
+ .bundle_lock
+
+## .bundle_align_mode needs a following integer value
+#--- mode-without-arg.s
+# CHECK: [[#@LINE+1]]:21: error: unknown token in expression
+ .bundle_align_mode
+ imull $17, %ebx, %ebp
+
+## .bundle_unlock can't come without a .bundle_lock before it
+#--- unlock-without-lock.s
+ .bundle_align_mode 3
+ imull $17, %ebx, %ebp
+# CHECK: [[#@LINE+1]]:3: error: .bundle_unlock without matching lock
+ .bundle_unlock
+
+## .bundle_lock can only take one `align_to_end` flag or no flag.
+#--- bad-lock-option.s
+ .bundle_align_mode 4
+# CHECK: [[#@LINE+1]]:16: error: invalid option for `.bundle_lock`
+ .bundle_lock 5
+ imull $17, %ebx, %ebp
+ .bundle_unlock
+
+## This test invokes .bundle_lock and then switches to a different section
+## w/o the appropriate unlock.
+#--- switch-section-locked.s
+ .bundle_align_mode 3
+ .section text1, "x"
+ imull $17, %ebx, %ebp
+ .bundle_lock
+ imull $17, %ebx, %ebp
+
+# CHECK: [[#@LINE+1]]:3: error: unterminated .bundle_lock
+ .section text2, "x"
+ imull $17, %ebx, %ebp
+
+## bundle lock size cannot be bigger than the align mode size
+#--- group-too-large.s
+ .text
+foo:
+ .bundle_align_mode 4
+ pushq %rbp
+
+ .bundle_lock
+ pushq %r14
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+# CHECK: [[#@LINE+1]]:3: error: fragment can't be larger than a bundle size
+ .bundle_unlock
+
+## test that nested lock is emitting the right error.
+#--- nested-lock.s
+ .bundle_align_mode 4
+foo:
+## bundle alignment mode can be set more than once.
+ .bundle_align_mode 4
+ .bundle_lock
+# CHECK: [[#@LINE+1]]:3: error: nested .bundle_lock is not allowed
+ .bundle_lock
+ .bundle_unlock
+ .bundle_unlock
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-inst.s b/llvm/test/MC/X86/AlignedBundling/bundle-inst.s
new file mode 100644
index 0000000000000..ebe7650156ef3
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-inst.s
@@ -0,0 +1,36 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test NOP padding to remove every instruction that crosses a bundle boundary.
+
+ .text
+foo:
+ .bundle_align_mode 5
+ ## 5 bytes * 6 = 30 bytes
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+
+## This imull is 3 bytes long and should have started at 0x1d, so two bytes
+## of nop padding are inserted instead and it starts at 0x20
+ imull $17, %ebx, %ebp
+# CHECK: 1e: nop
+# CHECK-NEXT: 20: imull
+
+## Sub-bundle .align with single-instruction bundling:
+## .align 16 is narrower than the 32-byte bundle; instructions after it
+## start mid-bundle and still receive NOP padding as needed.
+ pushq %rbp
+ .align 16
+ movl $1, (%rsp) ## 7 bytes at offset 16 — no padding needed
+ movl $2, 4(%rsp) ## 8 bytes; offset 23 → 23+8=31, fits in bundle
+ movl $3, (%rsp) ## 7 bytes at offset 31 → crosses boundary, pad to 32
+# CHECK: 30: mov
+# CHECK-NEXT: 37: mov
+# CHECK-NEXT: 3f: nop
+# CHECK-NEXT: 40: mov
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-lock.s b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
new file mode 100644
index 0000000000000..fca55509e3f7a
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
@@ -0,0 +1,57 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test NOP padding for bundle-locked groups.
+
+ .text
+foo:
+ .bundle_align_mode 4
+
+## Each of these callq instructions is 5 bytes long
+ callq bar
+ callq bar
+
+ .bundle_lock
+ callq bar
+ callq bar
+ .bundle_unlock
+## We'll need a 6-byte NOP before this group
+# CHECK: a: nop
+# CHECK-NEXT: 10: callq
+# CHECK-NEXT: 15: callq
+
+ .bundle_lock
+ callq bar
+ callq bar
+ .bundle_unlock
+## Same here
+# CHECK: 1a: nop
+# CHECK-NEXT: 20: callq
+# CHECK-NEXT: 25: callq
+
+ .align 16, 0x90
+lock_to_next_bundle:
+ callq bar
+ .bundle_lock
+ callq bar
+ callq bar
+ callq bar
+ .bundle_unlock
+## And here we'll need a 10-byte NOP + 1-byte NOP
+# CHECK: 30: callq
+# CHECK: 35: nop
+# CHECK-NEXT: 40: callq
+# CHECK-NEXT: 45: callq
+
+ .align 16, 0x90
+lock_fit_exactly:
+## offset=6, group=10 bytes (5 x 2): 6+10=16 == BUNDLE_SIZE
+ .fill 6, 1, 0x90
+ .bundle_lock
+ .rept 5
+ inc %eax
+ .endr
+ .bundle_unlock
+# CHECK: 56: incl
diff --git a/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
new file mode 100644
index 0000000000000..be13274dc8a9b
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
@@ -0,0 +1,40 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 --mattr=+fast-15bytenop %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test that long nops are generated for padding where possible while each respects the bundle align boundary.
+
+ .text
+foo:
+ .bundle_align_mode 5
+
+## This callq instruction is 5 bytes long
+ .bundle_lock align_to_end
+ callq bar
+ .bundle_unlock
+## To align this group to a bundle end, we need a 15-byte NOPs and a 12-byte NOP.
+# CHECK: 0: nop
+# CHECK-NEXT: f: nop
+# CHECK-NEXT: 1b: callq
+
+## This push instruction is 1 byte long
+ .bundle_lock align_to_end
+ push %rax
+ .bundle_unlock
+## To align this group to a bundle end, we need two 15-byte NOPs and a 1-byte NOP.
+# CHECK: 20: nop
+# CHECK-NEXT: 2f: nop
+# CHECK-NEXT: 3e: nop
+# CHECK-NEXT: 3f: pushq
+
+## bundle-aware optimization for `.nops N`
+ .p2align 5
+just_nops:
+ callq bar
+ .nops 64
+# CHECK: 40: callq
+# CHECK-NEXT: 45: nop
+# CHECK-NEXT: 54: nop
+# CHECK-NEXT: 60: nop
+# CHECK-NEXT: 6f: nop
+# CHECK-NEXT: 7e: nop
+# CHECK-NEXT: 80: nop
diff --git a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
new file mode 100644
index 0000000000000..b0d6d60a08642
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
@@ -0,0 +1,69 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - --x86-pad-max-prefix-size=5 \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+ .text
+add_prefix_prev:
+ .bundle_align_mode 5
+## This callq instruction is 5 bytes long
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+ .bundle_lock align_to_end
+ callq bar
+ .bundle_unlock
+# CHECK: 0: call
+# CHECK-NEXT: 5: call
+# CHECK-NEXT: a: call
+# CHECK-NEXT: 11: call
+# CHECK-NEXT: 1b: call
+
+ .p2align 5
+add_prefix_prev_next:
+ callq bar
+ .bundle_lock align_to_end
+ ## instructions inside a bundle lock can also be prefix-padded.
+ callq bar
+ .bundle_unlock
+# CHECK: 20: call
+# CHECK-NEXT: 2a: nop
+# CHECK: 36: call
+
+ .p2align 5
+ignore_nop_for_p2align:
+ int3
+ int3
+ ## no prefix padding with this 14-byte nop.
+ .p2align 4
+ int3
+ .bundle_lock
+ int3
+ .bundle_unlock
+ int3
+# CHECK: 40: int3
+# CHECK-NEXT: 41: int3
+# CHECK-NEXT: 42: nop
+# CHECK: 50: int3
+# CHECK-NEXT: 51: int3
+# CHECK-NEXT: 52: int3
+# CHECK-NEXT: 53: nop
+
+ .p2align 5
+ignore_nop_for_p2align5:
+ callq bar
+ .p2align 5
+.L1:
+ callq bar
+# CHECK: 60: call
+# CHECK-NEXT: 65: nop
+# CHECK: 80: call
+
+## ensure the last instructions are not prefix-padded
+ .p2align 5
+tail_bundle:
+ .bundle_lock
+ callq bar
+ .bundle_unlock
+ nop
+# CHECK: a0: call
+# CHECK-NEXT: a5: nop
diff --git a/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
new file mode 100644
index 0000000000000..5161bf4912c0b
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
@@ -0,0 +1,48 @@
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o - --x86-pad-max-prefix-size=1 \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+# This test checks whether enabling prefix padding with bundling can properly relax instructions proactively, avoiding fixup value overflows.
+
+ .text
+ .bundle_align_mode 5
+ .p2align 5
+
+ .rept 3
+ callq x ## 5 bytes each
+ .endr
+# CHECK: f: jmp
+# CHECK-NEXT: 14: int3
+ jmp near_target ## 2 bytes (rel8), has to be relaxed to 5 bytes (rel32)
+ .rept 8
+ int3 ## 1 byte each, 7 of them are prefix-padded
+ .endr
+ ## trailing NOPs are only consumed by prefix-padding the int3 instructions
+# CHECK: 1e: int3
+# CHECK-NEXT: 20: int3
+
+ ## Three full bundles of spacer to push near_target close to rel8 max range
+ .rept 3
+ .bundle_lock
+ .rept 32
+ int3
+ .endr
+ .bundle_unlock
+ .endr
+
+ .rept 15
+ int3
+ .endr
+ ## With prefix padding (max 1): instructions absorb trailing NOPs,
+ ## near_target shifts from 0x8f to 0x9e.
+ ## jmp distance = 0x9d - (0xf + len(jmp)) = 0x8c (140), exceeds rel8 range,
+ ## forcing relaxation to rel32.
+# CHECK: <near_target>:
+# CHECK-NEXT: 9d: inc
+near_target:
+ inc %eax
+
+ .bundle_lock
+ .rept 32
+ int3
+ .endr
+ .bundle_unlock
diff --git a/llvm/test/MC/X86/AlignedBundling/section.s b/llvm/test/MC/X86/AlignedBundling/section.s
new file mode 100644
index 0000000000000..fcd636cf2528d
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/section.s
@@ -0,0 +1,49 @@
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/two-sections.s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/two-sections.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/two-sections.s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/two-sections.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/section-alignment.s -o - \
+# RUN: | llvm-readobj --sections - | FileCheck %t/section-alignment.s
+
+## Test two different executable sections with bundling.
+#--- two-sections.s
+ .bundle_align_mode 3
+ .section text1, "x"
+# CHECK: section text1
+ imull $17, %ebx, %ebp
+ imull $17, %ebx, %ebp
+
+ imull $17, %ebx, %ebp
+# CHECK: 6: nop
+# CHECK-NEXT: 8: imull
+
+ .section text2, "x"
+# CHECK: section text2
+ imull $17, %ebx, %ebp
+ imull $17, %ebx, %ebp
+
+ imull $17, %ebx, %ebp
+# CHECK: 6: nop
+# CHECK-NEXT: 8: imull
+
+## Test that bundle-aligned sections with instructions are aligned
+#--- section-alignment.s
+ .bundle_align_mode 5
+# CHECK: Sections
+## Check that the empty .text section has the default alignment
+# CHECK-LABEL: Name: .text
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 4
+
+ .section text1, "x"
+ imull $17, %ebx, %ebp
+# CHECK-LABEL: Name: text1
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 32
+
+ .section text2, "x"
+ imull $17, %ebx, %ebp
+# CHECK-LABEL: Name: text2
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 32
>From 933878741d78f3ba9cc8aa22b20f44554f5eb46a Mon Sep 17 00:00:00 2001
From: Taehyun Noh <taehyun at utexas.edu>
Date: Fri, 24 Apr 2026 13:12:30 -0500
Subject: [PATCH 02/24] Correctly capture return value from padInstsBackward
---
llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 72affa7289ce3..48016006f0008 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -1113,7 +1113,7 @@ bool X86AsmBackend::finishLayout() const {
// of the resulting code. If we later find a reason to expand
// particular instructions over others, we can adjust.
unsigned RemainingSize = Asm->computeFragmentSize(F) - F.getFixedSize();
- padInstsBackward(Relaxable, RemainingSize);
+ Changed |= padInstsBackward(Relaxable, RemainingSize);
// If we're looking at a boundary align, make sure we don't try to pad
// its target instructions for some following directive. Doing so would
>From c712c250f1933093667c7bb0370b5c3c00fb8a76 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Wed, 13 May 2026 16:17:52 -0700
Subject: [PATCH 03/24] Restrict bundle alignment to executable sections
---
llvm/lib/MC/MCELFStreamer.cpp | 4 +++-
llvm/test/MC/X86/AlignedBundling/section.s | 25 ++++++++++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index e943711fb9de9..551f1e8984887 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -93,7 +93,9 @@ void MCELFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
}
// Ensure the previous section gets aligned if necessary.
- if (Asm.isBundlingEnabled() && CF->getParent()->hasInstructions())
+ auto *NewSectionELF = static_cast<const MCSectionELF *>(Section);
+ if (Asm.isBundlingEnabled() && CF->getParent()->hasInstructions() &&
+ (NewSectionELF->getFlags() & ELF::SHF_EXECINSTR))
Section->ensureMinAlignment(Align(Asm.getBundleAlignSize()));
}
auto *SectionELF = static_cast<const MCSectionELF *>(Section);
diff --git a/llvm/test/MC/X86/AlignedBundling/section.s b/llvm/test/MC/X86/AlignedBundling/section.s
index fcd636cf2528d..2a15345e6f68a 100644
--- a/llvm/test/MC/X86/AlignedBundling/section.s
+++ b/llvm/test/MC/X86/AlignedBundling/section.s
@@ -5,6 +5,8 @@
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/two-sections.s
# RUN: llvm-mc -filetype=obj -triple x86_64 %t/section-alignment.s -o - \
# RUN: | llvm-readobj --sections - | FileCheck %t/section-alignment.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/data-section-alignment.s -o - \
+# RUN: | llvm-readobj --sections - | FileCheck %t/data-section-alignment.s
## Test two different executable sections with bundling.
#--- two-sections.s
@@ -47,3 +49,26 @@
# CHECK-LABEL: Name: text2
# CHECK-NOT: Name
# CHECK: AddressAlignment: 32
+
+## Test that bundle alignment is only applied to executable sections.
+#--- data-section-alignment.s
+ .bundle_align_mode 5
+ .text
+ imull $17, %ebx, %ebp
+# CHECK-LABEL: Name: .text
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 32
+
+ .section .init_array,"aw"
+ .p2align 3
+ .quad 0
+# CHECK-LABEL: Name: .init_array
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 8
+
+ .section .data.rel.ro,"aw"
+ .p2align 3
+ .quad 0
+# CHECK-LABEL: Name: .data.rel.ro
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 8
>From 299fd0ae9b39aef6a980dd66029dc7d168189dae Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Sun, 14 Jun 2026 17:54:21 -0700
Subject: [PATCH 04/24] Simplify
---
llvm/lib/MC/MCAssembler.cpp | 14 +++++------
llvm/lib/MC/MCELFStreamer.cpp | 2 +-
llvm/lib/MC/MCObjectStreamer.cpp | 23 ++++++++-----------
.../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 11 +++------
4 files changed, 20 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 16c32f1de2732..95c82befdd485 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -1010,17 +1010,15 @@ MCAssembler::computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) {
Align BoundaryAlignment = BF.getAlignment();
uint64_t NewSize = 0;
- if (isBundlingEnabled()) {
- // For bundle alignment, we only pad instructions that cross the boundary.
- NewSize = mayCrossBoundary(AlignedOffset, AlignedSize, BoundaryAlignment)
+ if (!isBundlingEnabled()) {
+ NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
? offsetToAlignment(AlignedOffset, BoundaryAlignment)
: 0U;
- if (BF.isAlignToEnd()) {
- NewSize =
- offsetToAlignment(AlignedOffset + AlignedSize, BoundaryAlignment);
- }
+ } else if (BF.isAlignToEnd()) {
+ NewSize = offsetToAlignment(AlignedOffset + AlignedSize, BoundaryAlignment);
} else {
- NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
+ // For bundle alignment, we only pad instructions that cross the boundary.
+ NewSize = mayCrossBoundary(AlignedOffset, AlignedSize, BoundaryAlignment)
? offsetToAlignment(AlignedOffset, BoundaryAlignment)
: 0U;
}
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 551f1e8984887..068ef83147a84 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -94,7 +94,7 @@ void MCELFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
// Ensure the previous section gets aligned if necessary.
auto *NewSectionELF = static_cast<const MCSectionELF *>(Section);
- if (Asm.isBundlingEnabled() && CF->getParent()->hasInstructions() &&
+ if (CF->getParent()->hasInstructions() &&
(NewSectionELF->getFlags() & ELF::SHF_EXECINSTR))
Section->ensureMinAlignment(Align(Asm.getBundleAlignSize()));
}
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index bc98902df6faf..e5b36c30e8b18 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -416,18 +416,19 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
MCAssembler &Assembler = getAssembler();
MCAsmBackend &Backend = Assembler.getBackend();
+ auto relaxToFixpoint = [&](MCInst I) {
+ while (Backend.mayNeedRelaxation(I.getOpcode(), I.getOperands(), STI))
+ Backend.relaxInstruction(I, STI);
+ return I;
+ };
+
// To enable better bundle-nop optimization, we emit every instruction
// as a relaxable fragment.
if (Assembler.isBundlingEnabled()) {
- if (Sec->isBundleLocked() || Assembler.getRelaxAll()) {
- MCInst Relaxed = Inst;
- while (Backend.mayNeedRelaxation(Relaxed.getOpcode(),
- Relaxed.getOperands(), STI))
- Backend.relaxInstruction(Relaxed, STI);
- emitInstToFragment(Relaxed, STI);
- } else {
+ if (Sec->isBundleLocked() || Assembler.getRelaxAll())
+ emitInstToFragment(relaxToFixpoint(Inst), STI);
+ else
emitInstToFragment(Inst, STI);
- }
return;
}
@@ -439,11 +440,7 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
// Otherwise, relax and emit it as data if RelaxAll is specified.
if (Assembler.getRelaxAll()) {
- MCInst Relaxed = Inst;
- while (Backend.mayNeedRelaxation(Relaxed.getOpcode(), Relaxed.getOperands(),
- STI))
- Backend.relaxInstruction(Relaxed, STI);
- emitInstToData(Relaxed, STI);
+ emitInstToData(relaxToFixpoint(Inst), STI);
return;
}
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 48016006f0008..656d12b669904 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -948,8 +948,6 @@ bool X86AsmBackend::padInstsBackward(SmallVectorImpl<MCFragment *> &Relaxable,
bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
ArrayRef<MCFragment *> Peephole) const {
bool Changed = false;
-
- // Last Fragment is either FT_Align or FT_BoundaryAlign
auto *LastF = Peephole.back();
unsigned RemainingSize =
Asm.computeFragmentSize(*LastF) - LastF->getFixedSize();
@@ -986,8 +984,7 @@ bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
}
if (FIB->getKind() == MCFragment::FT_Relaxable) {
- auto &RF = cast<MCFragment>(*FIB);
- Relaxable.push_back(&RF);
+ Relaxable.push_back(FIB);
continue;
}
}
@@ -1015,10 +1012,8 @@ bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
RemainingSize += NextRemainingSize;
}
- // FT_Align sizes will be recalculated by layoutSection(),
- // FT_BoundaryAlign sizes are adjusted here.
- if (auto *BF = dyn_cast<MCBoundaryAlignFragment>(LastF))
- BF->setSize(RemainingSize);
+ // Record the computed padding on the BoundaryAlignFragment.
+ cast<MCBoundaryAlignFragment>(LastF)->setSize(RemainingSize);
return Changed;
}
>From 193fc7ae542a59b909ed92bbb95db1727ad27f64 Mon Sep 17 00:00:00 2001
From: Taehyun Noh <taehyun at utexas.edu>
Date: Mon, 15 Jun 2026 13:48:52 -0500
Subject: [PATCH 05/24] Address minor style comments
Remove excessive blank lines
Minor
---
llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 11 +++--------
.../AlignedBundling/asm-printing-bundle-directives.s | 2 +-
llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s | 2 +-
.../MC/X86/AlignedBundling/bundle-align-to-end-lock.s | 4 ++--
llvm/test/MC/X86/AlignedBundling/bundle-inst.s | 4 ++--
llvm/test/MC/X86/AlignedBundling/bundle-lock.s | 4 ++--
llvm/test/MC/X86/AlignedBundling/long-nop-pad.s | 2 +-
llvm/test/MC/X86/AlignedBundling/prefix-padding.s | 2 +-
.../MC/X86/AlignedBundling/relax-for-prefix-padding.s | 2 +-
llvm/test/MC/X86/AlignedBundling/section.s | 8 ++++----
10 files changed, 18 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 656d12b669904..a6e901f798541 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -500,18 +500,15 @@ void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
/// fragment and start a new empty fragment.
void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
assert(Asm->isBundlingEnabled());
-
MCFragment *CF = OS.getCurrentFragment();
- if (OS.getCurrentSectionOnly()->isBundleLocked()) {
- // We're still inside the lock, do not close the current fragment with BA.
+ // We're still inside the lock, do not close the current fragment with BA.
+ if (OS.getCurrentSectionOnly()->isBundleLocked())
return;
- }
assert(PendingBA && "MCBoundaryAlignFragment is expected for every "
"instruction if it is not bundle-locked");
PendingBA = nullptr;
-
CF->getParent()->ensureMinAlignment(Align(Asm->getBundleAlignSize()));
}
@@ -1025,9 +1022,7 @@ bool X86AsmBackend::optimizeBundleNops(const MCAssembler &Asm) const {
continue;
SmallVector<MCFragment *, 4> Bundle;
- for (MCSection::iterator I = Sec.begin(), IE = Sec.end(); I != IE; ++I) {
- MCFragment &F = *I;
-
+ for (MCFragment &F : Sec) {
if (F.getKind() == llvm::MCFragment::FT_BoundaryAlign) {
unsigned RemainingSize = Asm.computeFragmentSize(F) - F.getFixedSize();
if (RemainingSize > 0) {
diff --git a/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s b/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
index fcd53711e656e..298fa75302f7a 100644
--- a/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
+++ b/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=asm -triple x86_64 %s -o - 2>&1 | FileCheck %s
+# RUN: llvm-mc -filetype=asm -triple x86_64 %s | FileCheck %s
## Just a simple test for the assembly emitter - making sure it emits back the
## bundling directives.
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s b/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
index 41803b6f8d38c..beaf5f91d6f47 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s \
# RUN: | llvm-objdump -d - | FileCheck %s
## Test that instructions inside bundle-locked groups are relaxed even if their
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s b/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
index ef3dddc2a4ce2..853e95b9f2ccc 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
-# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
## Test NOP padding for `align_to_end` bundle-locked groups.
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-inst.s b/llvm/test/MC/X86/AlignedBundling/bundle-inst.s
index ebe7650156ef3..0ab68a5751198 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-inst.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-inst.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
-# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
## Test NOP padding to remove every instruction that crosses a bundle boundary.
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-lock.s b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
index fca55509e3f7a..1e7bbeb4ee73a 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
-# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
## Test NOP padding for bundle-locked groups.
diff --git a/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
index be13274dc8a9b..0351e32de13dc 100644
--- a/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
+++ b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 --mattr=+fast-15bytenop %s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 --mattr=+fast-15bytenop %s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
## Test that long nops are generated for padding where possible while each respects the bundle align boundary.
diff --git a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
index b0d6d60a08642..c9a1d47eac3a8 100644
--- a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
+++ b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - --x86-pad-max-prefix-size=5 \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s --x86-pad-max-prefix-size=5 \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
.text
diff --git a/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
index 5161bf4912c0b..56a08a99dc566 100644
--- a/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
+++ b/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o - --x86-pad-max-prefix-size=1 \
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s --x86-pad-max-prefix-size=1 \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
# This test checks whether enabling prefix padding with bundling can properly relax instructions proactively, avoiding fixup value overflows.
diff --git a/llvm/test/MC/X86/AlignedBundling/section.s b/llvm/test/MC/X86/AlignedBundling/section.s
index 2a15345e6f68a..5be4acf5cb9d2 100644
--- a/llvm/test/MC/X86/AlignedBundling/section.s
+++ b/llvm/test/MC/X86/AlignedBundling/section.s
@@ -1,11 +1,11 @@
# RUN: split-file %s %t
-# RUN: llvm-mc -filetype=obj -triple x86_64 %t/two-sections.s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/two-sections.s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/two-sections.s
-# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/two-sections.s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/two-sections.s \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/two-sections.s
-# RUN: llvm-mc -filetype=obj -triple x86_64 %t/section-alignment.s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/section-alignment.s \
# RUN: | llvm-readobj --sections - | FileCheck %t/section-alignment.s
-# RUN: llvm-mc -filetype=obj -triple x86_64 %t/data-section-alignment.s -o - \
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/data-section-alignment.s \
# RUN: | llvm-readobj --sections - | FileCheck %t/data-section-alignment.s
## Test two different executable sections with bundling.
>From e546db0dc12e10003291e0dbc57e9471a3aefac3 Mon Sep 17 00:00:00 2001
From: Taehyun Noh <taehyun at utexas.edu>
Date: Mon, 15 Jun 2026 14:16:27 -0500
Subject: [PATCH 06/24] Add untested corner cases
Change a test to have `.p2align` within a peephole
Add a CrossBoundary test case
---
.../MC/X86/AlignedBundling/bundle-errors.s | 6 +-
.../MC/X86/AlignedBundling/prefix-padding.s | 57 +++++++++++++------
2 files changed, 45 insertions(+), 18 deletions(-)
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
index 70e9795b74363..b593087ab42cb 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
@@ -14,6 +14,8 @@
imull $17, %ebx, %ebp
# CHECK: [[#@LINE+1]]:3: error: .bundle_lock forbidden when bundling is disabled
.bundle_lock
+# CHECK: [[#@LINE+1]]:3: error: .bundle_unlock forbidden when bundling is disabled
+ .bundle_unlock
## .bundle_align_mode needs a following integer value
#--- mode-without-arg.s
@@ -69,8 +71,10 @@ foo:
#--- nested-lock.s
.bundle_align_mode 4
foo:
-## bundle alignment mode can be set more than once.
+## repeating .bundle_align_mode with the same value is allowed.
.bundle_align_mode 4
+# CHECK: [[#@LINE+1]]:3: error: .bundle_align_mode cannot be changed once set
+ .bundle_align_mode 5
.bundle_lock
# CHECK: [[#@LINE+1]]:3: error: nested .bundle_lock is not allowed
.bundle_lock
diff --git a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
index c9a1d47eac3a8..9850d8ec581f5 100644
--- a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
+++ b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
@@ -1,7 +1,7 @@
# RUN: llvm-mc -filetype=obj -triple x86_64 %s --x86-pad-max-prefix-size=5 \
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
- .text
+ .section text1, "x"
add_prefix_prev:
.bundle_align_mode 5
## This callq instruction is 5 bytes long
@@ -32,23 +32,17 @@ add_prefix_prev_next:
.p2align 5
ignore_nop_for_p2align:
int3
- int3
- ## no prefix padding with this 14-byte nop.
+ ## no prefix padding with this 15-byte nop.
.p2align 4
- int3
- .bundle_lock
- int3
+ ## prefix padding with only callq, not the early int3
+ .bundle_lock align_to_end
+ callq bar
.bundle_unlock
- int3
# CHECK: 40: int3
-# CHECK-NEXT: 41: int3
-# CHECK-NEXT: 42: nop
-# CHECK: 50: int3
-# CHECK-NEXT: 51: int3
-# CHECK-NEXT: 52: int3
-# CHECK-NEXT: 53: nop
+# CHECK-NEXT: 41: nop
+# CHECK-NEXT: 50: nop
+# CHECK-NEXT: 56: call
- .p2align 5
ignore_nop_for_p2align5:
callq bar
.p2align 5
@@ -58,12 +52,41 @@ ignore_nop_for_p2align5:
# CHECK-NEXT: 65: nop
# CHECK: 80: call
+ .section text2, "x"
## ensure the last instructions are not prefix-padded
- .p2align 5
tail_bundle:
.bundle_lock
callq bar
.bundle_unlock
nop
-# CHECK: a0: call
-# CHECK-NEXT: a5: nop
+# CHECK: 0: call
+# CHECK-NEXT: 5: nop
+
+## Without prefix padding, the align_to_end group creates a 24-byte nop that
+## spans a bundle boundary, out of a single BoundaryAlign Fragment. This test
+## case ensures the 24-byte nop is consumed without overflowing any bundle.
+ .section text3, "x"
+ensure_bundle_boundary:
+ ## 20-byte group (5-byte * 4)
+ .bundle_lock
+ .rept 4
+ callq foo
+ .endr
+ .bundle_unlock
+ ## consume 12-byte by prefix-padding the first group (back-to-front).
+ ## consume another 12-byte by prefix-padding the second group (front-to-back).
+ .bundle_lock align_to_end
+ ## Although a maximum prefix budget is 20 bytes from this group,
+ ## only 12 bytes will be used.
+ .rept 4
+ callq bar
+ .endr
+ .bundle_unlock
+# CHECK: 0: call
+# CHECK-NEXT: 5: call
+# CHECK-NEXT: c: call
+# CHECK-NEXT: 16: call
+# CHECK-NEXT: 20: call
+# CHECK-NEXT: 2a: call
+# CHECK-NEXT: 34: call
+# CHECK-NEXT: 3b: call
>From d1d8d5d8dfc493cdf13e4cd7cdd38dd48c761cc2 Mon Sep 17 00:00:00 2001
From: Taehyun Noh <taehyun at utexas.edu>
Date: Mon, 15 Jun 2026 20:58:30 -0500
Subject: [PATCH 07/24] Fix bundling and prefix padding behavior around
prefix-only instructions
Previously, a prefix-only instruction could end up in a different bundle
than the following instruction. Additionally, such instructions could be
padded before and after, which distorts the original instruction's
semantics. This commit allows the bundling path to use canPadInst, which
disables prefix padding for a selected set of instructions. It also
introduces an implicit lock between a prefix and a suffix instruction,
ensuring that they are emitted into the same bundle.
---
.../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 40 +++++++++++++------
.../MC/X86/AlignedBundling/prefix-padding.s | 38 +++++++++++++++++-
2 files changed, 63 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index a6e901f798541..98d41534b7dc0 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -124,6 +124,7 @@ class X86AsmBackend : public MCAsmBackend {
Align AlignBoundary;
unsigned TargetPrefixMax = 0;
+ bool ReuseBA;
MCInst PrevInst;
unsigned PrevInstOpcode = 0;
MCBoundaryAlignFragment *PendingBA = nullptr;
@@ -474,12 +475,18 @@ void X86_MC::emitInstruction(MCObjectStreamer &S, const MCInst &Inst,
/// If the upcoming instruction is inside the bundle lock, do nothing so that
/// the ObjectStreamer emits the instruction to the current fragment. If not, it
-/// creates a new BA to group bundled fragments.
+/// creates a new BA to group bundled fragments. If PrevInst is just a prefix,
+/// we can reuse old BA.
void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
assert(Asm->isBundlingEnabled());
- if (OS.getCurrentSectionOnly()->isBundleLocked()) {
- OS.getCurrentFragment()->setAllowAutoPadding(true);
+ if (OS.getCurrentSectionOnly()->isBundleLocked())
+ return;
+ // when there is a prefix MCInst not locked, we need an implicit lock between
+ // the prefix and the next MCInst.
+ if (ReuseBA && PendingBA &&
+ PendingBA->getLastFragment()->getParent() == OS.getCurrentSectionOnly()) {
+ PendingBA->setLastFragment(OS.getCurrentFragment());
return;
}
PendingBA = OS.newSpecialFragment<MCBoundaryAlignFragment>(
@@ -490,14 +497,13 @@ void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
// emitCodeAlignment repurposes in-place to FT_Align, corrupting the BA's
// boundary range.
PendingBA->setLastFragment(OS.getCurrentFragment());
-
- OS.getCurrentFragment()->setAllowAutoPadding(true);
}
/// If the just-emitted instruction is inside the bundle lock, check the current
-/// fragment is non-zero to ensure the instruction is placed as expected. If it
-/// is not locked, finalize pending BA Fragment. emitBundleUnlock will close the
-/// fragment and start a new empty fragment.
+/// fragment is non-zero to ensure the instruction is placed as expected.
+/// emitBundleUnlock will close the fragment and start a new empty fragment. If
+/// it is not locked, finalize pending BA Fragment. If it is a prefix, let the
+/// next instruction reuse the same BA.
void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
assert(Asm->isBundlingEnabled());
MCFragment *CF = OS.getCurrentFragment();
@@ -508,17 +514,25 @@ void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
assert(PendingBA && "MCBoundaryAlignFragment is expected for every "
"instruction if it is not bundle-locked");
- PendingBA = nullptr;
CF->getParent()->ensureMinAlignment(Align(Asm->getBundleAlignSize()));
+
+ // Update ReuseBA for the next BeginBundle.
+ ReuseBA = isPrefix(PrevInstOpcode, *MCII);
+ if (ReuseBA)
+ return;
+ PendingBA = nullptr;
}
/// Insert BoundaryAlignFragment before instructions to align branches.
void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
const MCInst &Inst,
const MCSubtargetInfo &STI) {
- if (Asm->isBundlingEnabled())
- return emitInstructionBeginBundle(OS);
bool CanPadInst = canPadInst(Inst, OS);
+ if (Asm->isBundlingEnabled()) {
+ emitInstructionBeginBundle(OS);
+ OS.getCurrentFragment()->setAllowAutoPadding(CanPadInst);
+ return;
+ }
if (CanPadInst)
OS.getCurrentFragment()->setAllowAutoPadding(true);
@@ -580,12 +594,12 @@ void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
/// Set the last fragment to be aligned for the BoundaryAlignFragment.
void X86AsmBackend::emitInstructionEnd(MCObjectStreamer &OS,
const MCInst &Inst) {
- if (Asm->isBundlingEnabled())
- return emitInstructionEndBundle(OS);
// Update PrevInstOpcode here, canPadInst() reads that.
MCFragment *CF = OS.getCurrentFragment();
PrevInstOpcode = Inst.getOpcode();
PrevInstPosition = std::make_pair(CF, OS.getCurFragSize());
+ if (Asm->isBundlingEnabled())
+ return emitInstructionEndBundle(OS);
if (!canPadBranches(OS))
return;
diff --git a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
index 9850d8ec581f5..7eafd73ce7654 100644
--- a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
+++ b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
@@ -1,6 +1,10 @@
-# RUN: llvm-mc -filetype=obj -triple x86_64 %s --x86-pad-max-prefix-size=5 \
-# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/prefix-pad.s --x86-pad-max-prefix-size=5 \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/prefix-pad.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/no-prefix-pad.s --x86-pad-max-prefix-size=1 \
+# RUN: | llvm-objdump -d - | FileCheck %t/no-prefix-pad.s
+#--- prefix-pad.s
.section text1, "x"
add_prefix_prev:
.bundle_align_mode 5
@@ -90,3 +94,33 @@ ensure_bundle_boundary:
# CHECK-NEXT: 2a: call
# CHECK-NEXT: 34: call
# CHECK-NEXT: 3b: call
+
+## This test contains instructions that must not be padded.
+#--- no-prefix-pad.s
+ .text
+ .bundle_align_mode 5
+prefix_cmpxchg_in_a_bundle:
+ movl 48(%rbx), %r12d
+ movl __thread_list_lock(%rip), %eax
+ cmpl %r12d, %eax
+ jne .LBB4_7
+ incl tl_lock_count(%rip)
+ jmp .LBB4_10
+ .LBB4_7:
+ xorl %eax, %eax
+ lock
+ cmpxchgl %r12d, __thread_list_lock(%rip)
+# CHECK: 1c: 2e 31 c0 xorl
+# CHECK-NEXT: 1f: 90 nop
+## lock must start the new bundle, not replacing the nop right above
+# CHECK-NEXT: 20: f0 lock
+# CHECK-NEXT: 21: 44 0f b1 25 00 00 00 00 cmpxchgl %r12d, (%rip)
+ .LBB4_10:
+ xor %rax, %rax
+
+no_pad_before_after_prefix:
+ .p2align 5
+ lock
+ cmpxchgl %r12d, __thread_list_lock(%rip)
+# CHECK: 40: f0 lock
+# CHECK-NEXT: 41: 44 0f b1 25 00 00 00 00 cmpxchgl %r12d, (%rip)
>From 740187eed2a4f64fde1d0fc575534d007c09a3b3 Mon Sep 17 00:00:00 2001
From: Taehyun Noh <taehyun at utexas.edu>
Date: Wed, 8 Jul 2026 21:39:04 -0500
Subject: [PATCH 08/24] Use `reportFatalInternalError` instead of
`report_fatal_error`
---
llvm/lib/MC/MCAssembler.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 95c82befdd485..099f87abe1e2d 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -416,7 +416,7 @@ static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
assert(NumBytesToEmit && "try to emit zero-sized NOP");
if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit, STI)) {
- report_fatal_error("unable to write NOP sequence of the remaining " +
+ reportFatalInternalError("unable to write NOP sequence of the remaining " +
Twine(NumBytesToEmit) + " bytes");
return;
}
@@ -583,7 +583,7 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
if (!Asm.isBundlingEnabled()) {
if (!Asm.getBackend().writeNopData(OS, FragmentSize,
BF.getSubtargetInfo()))
- report_fatal_error("unable to write nop sequence of " +
+ reportFatalInternalError("unable to write nop sequence of " +
Twine(FragmentSize) + " bytes");
} else {
writeControlledNops(OS, Asm, FragmentSize, Asm.getFragmentOffset(BF),
>From 8640766f8d5a43449b9fb317bd1384bdb1d840e0 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Thu, 9 Jul 2026 00:43:46 -0700
Subject: [PATCH 09/24] Apply clang-format fix
---
llvm/lib/MC/MCAssembler.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 099f87abe1e2d..37a08337b7498 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -416,8 +416,9 @@ static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
assert(NumBytesToEmit && "try to emit zero-sized NOP");
if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit, STI)) {
- reportFatalInternalError("unable to write NOP sequence of the remaining " +
- Twine(NumBytesToEmit) + " bytes");
+ reportFatalInternalError(
+ "unable to write NOP sequence of the remaining " +
+ Twine(NumBytesToEmit) + " bytes");
return;
}
@@ -584,7 +585,7 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
if (!Asm.getBackend().writeNopData(OS, FragmentSize,
BF.getSubtargetInfo()))
reportFatalInternalError("unable to write nop sequence of " +
- Twine(FragmentSize) + " bytes");
+ Twine(FragmentSize) + " bytes");
} else {
writeControlledNops(OS, Asm, FragmentSize, Asm.getFragmentOffset(BF),
FragmentSize, BF.getSubtargetInfo());
>From 9df4ad9cfee522baf94d323285d7156f4c78ea38 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Sun, 19 Jul 2026 23:50:44 -0700
Subject: [PATCH 10/24] Add documentation for AlignedBundling
---
llvm/docs/AlignedBundling.rst | 97 +++++++++++++++++++++++++++++++++++
llvm/docs/Reference.md | 5 ++
2 files changed, 102 insertions(+)
create mode 100644 llvm/docs/AlignedBundling.rst
diff --git a/llvm/docs/AlignedBundling.rst b/llvm/docs/AlignedBundling.rst
new file mode 100644
index 0000000000000..23100ad7d6add
--- /dev/null
+++ b/llvm/docs/AlignedBundling.rst
@@ -0,0 +1,97 @@
+============================
+Aligned Instruction Bundling
+============================
+
+Overview
+========
+
+*Aligned instruction bundling* partitions the instructions in a section into
+fixed-size, naturally aligned groups called bundles, and guarantees that no
+instruction ever crosses a bundle boundary. Consecutive instructions can also
+be grouped so that the assembler guarantees the whole group resides within a
+single bundle.
+
+Bundling is a building block for software-based fault isolation (SFI) and
+sandboxing schemes. Forcing every instruction to begin at one of a statically
+known set of offsets gives the instruction stream a single canonical decoding:
+control flow cannot jump into the middle of an instruction to manufacture a
+different, unchecked instruction sequence. When combined with masking of
+indirect branch targets to bundle-aligned addresses, this constrains all
+control flow to a statically verifiable set of locations and instructions.
+Bundling is used by the x86-64 implementation of :doc:`Lightweight Fault
+Isolation (LFI) <LFI>`.
+
+.. note::
+
+ The current LLVM implementation supports bundling only for x86 ELF targets.
+
+``.bundle_align_mode``
+======================
+
+::
+
+ .bundle_align_mode abs-expr
+
+Enables aligned bundle mode and sets the bundle size to ``2^abs-expr`` bytes,
+where ``abs-expr`` is a power-of-two exponent between 0 and 30 (as for the
+``.p2align`` directive). For example, ``.bundle_align_mode 5`` selects 32-byte
+bundles.
+
+While bundling is enabled, the assembler ensures that no single instruction
+spans a boundary between two bundles. When an instruction would not fit in the
+space remaining in the current bundle, that space is filled with no-op
+instructions so the instruction starts at the beginning of the next bundle.
+
+Enabling bundle mode also raises the alignment of every text section that
+receives instructions to at least the bundle size.
+
+Once enabled, bundle mode stays in effect for the rest of the file and its
+bundle size is fixed.
+
+``.bundle_lock`` and ``.bundle_unlock``
+=======================================
+
+::
+
+ .bundle_lock [align_to_end]
+ ...instructions...
+ .bundle_unlock
+
+A ``.bundle_lock`` / ``.bundle_unlock`` pair encloses a sequence of
+instructions that must all be placed in a single bundle. The assembler inserts
+padding before the sequence, if necessary, so that the entire group lands
+within one bundle rather than straddling a boundary.
+
+The enclosed sequence must fit within a single bundle -- it is an error if the
+total size of the locked instructions exceeds the bundle size.
+
+Both directives are only valid after bundle mode has been enabled with
+``.bundle_align_mode``. A ``.bundle_unlock`` must be matched by a preceding
+``.bundle_lock`` in the same section, and a section may not be switched while a
+``.bundle_lock`` is open.
+
+``align_to_end``
+----------------
+
+By default a locked group is padded at the front so that it starts far enough
+into the bundle to fit. With the ``align_to_end`` option the group is instead
+padded so that its last instruction ends exactly on a bundle boundary.
+
+Padding
+=======
+
+By default, bundle padding is emitted as no-op instructions, and neither an
+instruction nor a padding no-op is ever allowed to cross a bundle boundary.
+
+Prefix padding (x86)
+--------------------
+
+On x86 the assembler can instead absorb some of the required padding into
+neighboring instructions by prepending otherwise-ignored instruction prefixes
+to them, avoiding standalone no-ops. This is controlled by:
+
+.. option:: --x86-pad-max-prefix-size=<N>
+
+ Maximum number of prefixes the assembler may add to an instruction for
+ padding. ``0`` (the default) disables prefix padding, so only no-op
+ instructions are used.
diff --git a/llvm/docs/Reference.md b/llvm/docs/Reference.md
index 24a3fe88106d4..b212067e919db 100644
--- a/llvm/docs/Reference.md
+++ b/llvm/docs/Reference.md
@@ -33,6 +33,7 @@ FaultMaps
Atomics
ExceptionHandling
Extensions
+AlignedBundling
HowToSetUpLLVMStyleRTTI
BlockFrequencyTerminology
BranchWeightMetadata
@@ -171,6 +172,10 @@ XRayFDRFormat
{doc}`Extensions`
: LLVM-specific extensions to tools and formats LLVM seeks compatibility with.
+{doc}`AlignedBundling`
+: The `.bundle_align_mode`, `.bundle_lock`, and `.bundle_unlock` assembler
+ directives.
+
{doc}`HowToSetUpLLVMStyleRTTI`
: How to make `isa<>`, `dyn_cast<>`, etc. available for clients of your
class hierarchy.
>From e5c95422e31b11b8f20e3ad066a98e0fd99d14d2 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Mon, 20 Jul 2026 10:52:37 -0700
Subject: [PATCH 11/24] Emit error for conflict between x86-align-branch and
bundling
---
llvm/lib/MC/MCELFStreamer.cpp | 10 ++++++++--
llvm/test/MC/X86/AlignedBundling/bundle-errors.s | 8 ++++++++
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 068ef83147a84..5244ae511e51d 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -342,11 +342,17 @@ void MCELFStreamer::emitBundleAlignMode(Align Alignment) {
setAllowAutoPadding(true);
if (Alignment > 1 && (Assembler.getBundleAlignSize() == 0 ||
- Assembler.getBundleAlignSize() == Alignment.value()))
+ Assembler.getBundleAlignSize() == Alignment.value())) {
+ if (Assembler.getBundleAlignSize() == 0 &&
+ Assembler.getBackend().allowAutoPadding())
+ getContext().reportError(
+ getStartTokLoc(),
+ ".bundle_align_mode is incompatible with branch alignment");
Assembler.setBundleAlignSize(Alignment.value());
- else
+ } else {
getContext().reportError(getStartTokLoc(),
".bundle_align_mode cannot be changed once set");
+ }
}
void MCELFStreamer::emitBundleLock(bool AlignToEnd,
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
index b593087ab42cb..30adc544cb58f 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
@@ -7,6 +7,8 @@
# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/nested-lock.s 2>&1 | FileCheck %t/nested-lock.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 -x86-branches-within-32B-boundaries %t/bundle-with-align-branch.s 2>&1 | FileCheck %t/bundle-with-align-branch.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 -x86-align-branch-boundary=32 -x86-align-branch=jmp %t/bundle-with-align-branch.s 2>&1 | FileCheck %t/bundle-with-align-branch.s
## .bundle_lock can't come without a .bundle_align_mode before it
@@ -80,3 +82,9 @@ foo:
.bundle_lock
.bundle_unlock
.bundle_unlock
+
+## Instruction bundling cannot be combined with branch alignment.
+#--- bundle-with-align-branch.s
+# CHECK: [[#@LINE+1]]:3: error: .bundle_align_mode is incompatible with branch alignment
+ .bundle_align_mode 5
+ imull $17, %ebx, %ebp
>From f7bb72bc3b5cd4cd7367ce2743210c67ca76d542 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Mon, 20 Jul 2026 11:19:52 -0700
Subject: [PATCH 12/24] Add relax-all bundling test
---
.../MC/X86/AlignedBundling/bundle-relax-all.s | 27 +++++++++++++++++++
1 file changed, 27 insertions(+)
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-relax-all.s
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-relax-all.s b/llvm/test/MC/X86/AlignedBundling/bundle-relax-all.s
new file mode 100644
index 0000000000000..f58a2714aee9a
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-relax-all.s
@@ -0,0 +1,27 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefix=SHORT
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefix=RELAX
+
+ .text
+ .bundle_align_mode 4
+foo:
+ .rept 12
+ int3
+ .endr
+ jle .Lskip
+.Lskip:
+ int3
+
+## Without relaxation the jump is 2 bytes at offset 0xc and does not cross the
+## 0x10 boundary, so no padding is inserted.
+# SHORT: b: int3
+# SHORT-NEXT: c: jle
+# SHORT-NEXT: e: int3
+
+## With -mc-relax-all the jump is 6 bytes, so a 4-byte NOP moves it to the
+## next bundle at 0x10.
+# RELAX: b: int3
+# RELAX-NEXT: c: nop
+# RELAX-NEXT: 10: jle
+# RELAX-NEXT: 16: int3
>From 276fe1ef7e182beab4b4652bf07ff8108aeb1920 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Mon, 20 Jul 2026 11:52:32 -0700
Subject: [PATCH 13/24] Avoid redundant finishLayout work when prefix padding
is disabled
---
llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 98d41534b7dc0..bcc0bb886bc5f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -1057,8 +1057,10 @@ bool X86AsmBackend::optimizeBundleNops(const MCAssembler &Asm) const {
}
bool X86AsmBackend::finishLayout() const {
- if (Asm->isBundlingEnabled() && TargetPrefixMax != 0)
- return optimizeBundleNops(*Asm);
+ // With bundling, padding is fully determined during layout and the only
+ // post-layout optimization is prefix padding.
+ if (Asm->isBundlingEnabled())
+ return TargetPrefixMax != 0 && optimizeBundleNops(*Asm);
// See if we can further relax some instructions to cut down on the number of
// nop bytes required for code alignment. The actual win is in reducing
// instruction count, not number of bytes. Modern X86-64 can easily end up
>From 979f6b85df0932e5eeaf07294c49282523690fe8 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Mon, 20 Jul 2026 11:56:14 -0700
Subject: [PATCH 14/24] Fix possible nop spanning bundle boundary from p2align
---
llvm/lib/MC/MCAssembler.cpp | 13 ++++++++++---
llvm/test/MC/X86/AlignedBundling/long-nop-pad.s | 17 +++++++++++++++++
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 37a08337b7498..3b011ea1cd061 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -471,9 +471,16 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
// In the nops mode, call the backend hook to write `Count` nops.
if (F.hasAlignEmitNops()) {
- if (!Asm.getBackend().writeNopData(OS, Count, F.getSubtargetInfo()))
- reportFatalInternalError("unable to write nop sequence of " +
- Twine(Count) + " bytes");
+ if (!Asm.isBundlingEnabled()) {
+ if (!Asm.getBackend().writeNopData(OS, Count, F.getSubtargetInfo()))
+ reportFatalInternalError("unable to write nop sequence of " +
+ Twine(Count) + " bytes");
+ } else {
+ // Ensure that no nop of the fill crosses a bundle boundary.
+ writeControlledNops(OS, Asm, Count,
+ Asm.getFragmentOffset(F) + F.getFixedSize(), Count,
+ F.getSubtargetInfo());
+ }
} else {
// Otherwise, write out in multiples of the value size.
for (uint64_t i = 0; i != Count; ++i) {
diff --git a/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
index 0351e32de13dc..125fba79dd852 100644
--- a/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
+++ b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
@@ -38,3 +38,20 @@ just_nops:
# CHECK-NEXT: 6f: nop
# CHECK-NEXT: 7e: nop
# CHECK-NEXT: 80: nop
+
+## The nop fill emitted for an alignment directive must also respect bundle
+## boundaries. An alignment larger than the bundle size produces a fill that
+## spans multiple bundles; no single nop of that fill may cross a boundary.
+ .p2align 6
+align_fill:
+ .rept 25
+ int3
+ .endr
+ .p2align 6
+ int3
+## The 39-byte fill from 0xd9 to 0x100 must break at the bundle boundary 0xe0.
+# CHECK: d9: nop
+# CHECK-NEXT: e0: nop
+# CHECK-NEXT: ef: nop
+# CHECK-NEXT: fe: nop
+# CHECK-NEXT: 100: int3
>From 8c59b969904d1ff82a0c61e18b23b21692f372ae Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Mon, 20 Jul 2026 13:26:55 -0700
Subject: [PATCH 15/24] Minor improvements in error reporting and comment
consistency
---
llvm/include/llvm/MC/MCStreamer.h | 5 +-
llvm/lib/MC/MCELFStreamer.cpp | 58 +++++++++----------
.../MC/X86/AlignedBundling/bundle-errors.s | 17 ++++++
3 files changed, 49 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index d7e212ffe7d70..d333bf4f9774d 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -1131,8 +1131,9 @@ class LLVM_ABI MCStreamer {
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym);
- /// Set the bundle alignment mode from now on in the section.
- /// The value 1 means turn the bundle alignment off.
+ /// Enable aligned instruction bundling with the given bundle size, from
+ /// this point onward. Once enabled, bundling cannot be disabled and the
+ /// bundle size cannot be changed.
virtual void emitBundleAlignMode(Align Alignment);
/// The following instructions are a bundle-locked group.
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 5244ae511e51d..837e6c2cd04d5 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -81,22 +81,13 @@ void MCELFStreamer::emitLabelAtPos(MCSymbol *S, SMLoc Loc, MCFragment &F,
void MCELFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
MCAssembler &Asm = getAssembler();
MCFragment *CF = getCurrentFragment();
- if (Asm.isBundlingEnabled()) {
- if (isBundleLocked()) {
- getContext().reportError(
- getStartTokLoc(),
- "unterminated .bundle_lock when changing a section");
- // Clean up bundle state to allow continuing.
- MCSection *CurSec = CF->getParent();
- CurSec->setIsBundleLocked(false);
- BundleBA = nullptr;
- }
-
- // Ensure the previous section gets aligned if necessary.
- auto *NewSectionELF = static_cast<const MCSectionELF *>(Section);
- if (CF->getParent()->hasInstructions() &&
- (NewSectionELF->getFlags() & ELF::SHF_EXECINSTR))
- Section->ensureMinAlignment(Align(Asm.getBundleAlignSize()));
+ if (Asm.isBundlingEnabled() && isBundleLocked()) {
+ getContext().reportError(getStartTokLoc(),
+ "unterminated .bundle_lock when changing a "
+ "section");
+ // Clean up bundle state to allow continuing.
+ CF->getParent()->setIsBundleLocked(false);
+ BundleBA = nullptr;
}
auto *SectionELF = static_cast<const MCSectionELF *>(Section);
const MCSymbol *Grp = SectionELF->getGroup();
@@ -336,23 +327,25 @@ void MCELFStreamer::emitIdent(StringRef IdentString) {
void MCELFStreamer::emitBundleAlignMode(Align Alignment) {
if (Log2(Alignment) > 30)
- getContext().reportError(getStartTokLoc(),
- ".bundle_align_mode alignment must be <= 30");
+ return getContext().reportError(
+ getStartTokLoc(), ".bundle_align_mode alignment must be <= 30");
+ if (Alignment == 1)
+ return getContext().reportError(
+ getStartTokLoc(), "disabling .bundle_align_mode is not supported");
MCAssembler &Assembler = getAssembler();
- setAllowAutoPadding(true);
-
- if (Alignment > 1 && (Assembler.getBundleAlignSize() == 0 ||
- Assembler.getBundleAlignSize() == Alignment.value())) {
- if (Assembler.getBundleAlignSize() == 0 &&
- Assembler.getBackend().allowAutoPadding())
- getContext().reportError(
- getStartTokLoc(),
- ".bundle_align_mode is incompatible with branch alignment");
- Assembler.setBundleAlignSize(Alignment.value());
- } else {
+ if (Assembler.getBundleAlignSize() != 0 &&
+ Assembler.getBundleAlignSize() != Alignment.value()) {
getContext().reportError(getStartTokLoc(),
".bundle_align_mode cannot be changed once set");
+ return;
}
+ if (Assembler.getBundleAlignSize() == 0 &&
+ Assembler.getBackend().allowAutoPadding())
+ getContext().reportError(
+ getStartTokLoc(),
+ ".bundle_align_mode is incompatible with branch alignment");
+ setAllowAutoPadding(true);
+ Assembler.setBundleAlignSize(Alignment.value());
}
void MCELFStreamer::emitBundleLock(bool AlignToEnd,
@@ -400,6 +393,13 @@ void MCELFStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
// Bundle overflow check.
uint64_t AlignedSize = 0;
for (const MCFragment *F = BundleBA->getNext();; F = F->getNext()) {
+ if (F->getKind() == MCFragment::FT_Align ||
+ F->getKind() == MCFragment::FT_Org) {
+ getContext().reportError(getStartTokLoc(),
+ "alignment and .org directives are not "
+ "supported inside a .bundle_lock group");
+ break;
+ }
AlignedSize += getAssembler().computeFragmentSize(*F);
if (F == BundleBA->getLastFragment())
break;
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
index 30adc544cb58f..e417198783bee 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
@@ -7,6 +7,8 @@
# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/nested-lock.s 2>&1 | FileCheck %t/nested-lock.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/mode-zero.s 2>&1 | FileCheck %t/mode-zero.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/align-in-lock.s 2>&1 | FileCheck %t/align-in-lock.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 -x86-branches-within-32B-boundaries %t/bundle-with-align-branch.s 2>&1 | FileCheck %t/bundle-with-align-branch.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 -x86-align-branch-boundary=32 -x86-align-branch=jmp %t/bundle-with-align-branch.s 2>&1 | FileCheck %t/bundle-with-align-branch.s
@@ -83,6 +85,21 @@ foo:
.bundle_unlock
.bundle_unlock
+## `.bundle_align_mode 0` is not supported.
+#--- mode-zero.s
+# CHECK: [[#@LINE+1]]:3: error: disabling .bundle_align_mode is not supported
+ .bundle_align_mode 0
+ imull $17, %ebx, %ebp
+
+#--- align-in-lock.s
+ .bundle_align_mode 4
+ .bundle_lock
+ incl %eax
+ .p2align 3
+ incl %eax
+# CHECK: [[#@LINE+1]]:3: error: alignment and .org directives are not supported inside a .bundle_lock group
+ .bundle_unlock
+
## Instruction bundling cannot be combined with branch alignment.
#--- bundle-with-align-branch.s
# CHECK: [[#@LINE+1]]:3: error: .bundle_align_mode is incompatible with branch alignment
>From 5d69f3bb2d689883aed15b3c4dc3cb529a654102 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Thu, 23 Jul 2026 11:29:05 -0700
Subject: [PATCH 16/24] Changes based on reviewer feedback
* Change from unsigned to MaybeAlign.
* Convert while loop to for loop.
* Restructure end of computeBoundaryAlignSize.
---
llvm/include/llvm/MC/MCAssembler.h | 13 ++++--
llvm/lib/MC/MCAssembler.cpp | 40 +++++++++----------
llvm/lib/MC/MCELFStreamer.cpp | 16 ++++----
.../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 17 ++++----
4 files changed, 42 insertions(+), 44 deletions(-)
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 2ede309321631..cf8612343aa74 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/iterator_range.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/SMLoc.h"
#include <cassert>
@@ -64,7 +65,8 @@ class MCAssembler {
// forward-reference displacements in `evaluateFixup`.
int64_t Stretch = 0;
- unsigned BundleAlignSize = 0;
+ /// Non-empty when aligned instruction bundling is enabled.
+ MaybeAlign BundleAlign;
SectionListType Sections;
@@ -203,9 +205,12 @@ class MCAssembler {
void setRelaxAll(bool Value) { RelaxAll = Value; }
int64_t getStretch() const { return Stretch; }
- bool isBundlingEnabled() const { return BundleAlignSize != 0; }
- unsigned getBundleAlignSize() const { return BundleAlignSize; }
- void setBundleAlignSize(unsigned Size) { BundleAlignSize = Size; }
+ bool isBundlingEnabled() const { return bool(BundleAlign); }
+ Align getBundleAlign() const {
+ assert(BundleAlign && "bundling is not enabled");
+ return *BundleAlign;
+ }
+ void setBundleAlign(Align Value) { BundleAlign = Value; }
const_iterator begin() const { return Sections.begin(); }
const_iterator end() const { return Sections.end(); }
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 3b011ea1cd061..07aa19587619e 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -401,15 +401,16 @@ static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
uint64_t NumBytes, uint64_t FragmentOffset,
uint64_t MaxNopSize,
const MCSubtargetInfo *STI) {
- uint64_t NumBytesEmitted = 0;
- while (NumBytesEmitted < NumBytes) {
- uint64_t NumBytesToEmit = std::min(NumBytes - NumBytesEmitted, MaxNopSize);
+ uint64_t NumBytesToEmit = 0;
+ for (uint64_t NumBytesEmitted = 0; NumBytesEmitted < NumBytes;
+ NumBytesEmitted += NumBytesToEmit) {
+ NumBytesToEmit = std::min(NumBytes - NumBytesEmitted, MaxNopSize);
if (Asm.isBundlingEnabled()) {
- unsigned BundleAlignSize = Asm.getBundleAlignSize();
+ uint64_t BundleSize = Asm.getBundleAlign().value();
uint64_t OffsetInBundle =
- (FragmentOffset + NumBytesEmitted) & (BundleAlignSize - 1);
- uint64_t SpaceInBundle = BundleAlignSize - OffsetInBundle;
+ (FragmentOffset + NumBytesEmitted) & (BundleSize - 1);
+ uint64_t SpaceInBundle = BundleSize - OffsetInBundle;
NumBytesToEmit = std::min(NumBytesToEmit, SpaceInBundle);
}
@@ -421,8 +422,6 @@ static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
Twine(NumBytesToEmit) + " bytes");
return;
}
-
- NumBytesEmitted += NumBytesToEmit;
}
}
@@ -1017,20 +1016,17 @@ MCAssembler::computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) {
Align BoundaryAlignment = BF.getAlignment();
- uint64_t NewSize = 0;
- if (!isBundlingEnabled()) {
- NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
- ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
- : 0U;
- } else if (BF.isAlignToEnd()) {
- NewSize = offsetToAlignment(AlignedOffset + AlignedSize, BoundaryAlignment);
- } else {
- // For bundle alignment, we only pad instructions that cross the boundary.
- NewSize = mayCrossBoundary(AlignedOffset, AlignedSize, BoundaryAlignment)
- ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
- : 0U;
- }
- return NewSize;
+ if (!isBundlingEnabled())
+ return needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
+ ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
+ : 0U;
+ if (BF.isAlignToEnd())
+ return offsetToAlignment(AlignedOffset + AlignedSize, BoundaryAlignment);
+
+ // For bundle alignment, we only pad instructions that cross the boundary.
+ return mayCrossBoundary(AlignedOffset, AlignedSize, BoundaryAlignment)
+ ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
+ : 0U;
}
void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 837e6c2cd04d5..589d3aae9b57f 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -333,19 +333,19 @@ void MCELFStreamer::emitBundleAlignMode(Align Alignment) {
return getContext().reportError(
getStartTokLoc(), "disabling .bundle_align_mode is not supported");
MCAssembler &Assembler = getAssembler();
- if (Assembler.getBundleAlignSize() != 0 &&
- Assembler.getBundleAlignSize() != Alignment.value()) {
+ if (Assembler.isBundlingEnabled() &&
+ Assembler.getBundleAlign() != Alignment) {
getContext().reportError(getStartTokLoc(),
".bundle_align_mode cannot be changed once set");
return;
}
- if (Assembler.getBundleAlignSize() == 0 &&
+ if (!Assembler.isBundlingEnabled() &&
Assembler.getBackend().allowAutoPadding())
getContext().reportError(
getStartTokLoc(),
".bundle_align_mode is incompatible with branch alignment");
setAllowAutoPadding(true);
- Assembler.setBundleAlignSize(Alignment.value());
+ Assembler.setBundleAlign(Alignment);
}
void MCELFStreamer::emitBundleLock(bool AlignToEnd,
@@ -366,9 +366,8 @@ void MCELFStreamer::emitBundleLock(bool AlignToEnd,
}
Sec.setIsBundleLocked(true);
- auto AlignBoundary = Asm.getBundleAlignSize();
BundleBA =
- newSpecialFragment<MCBoundaryAlignFragment>(Align(AlignBoundary), STI);
+ newSpecialFragment<MCBoundaryAlignFragment>(Asm.getBundleAlign(), STI);
BundleBA->setAlignToEnd(AlignToEnd);
}
@@ -406,14 +405,13 @@ void MCELFStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
}
BundleBA = nullptr;
- if (AlignedSize > getAssembler().getBundleAlignSize())
+ if (AlignedSize > getAssembler().getBundleAlign().value())
getContext().reportError(getStartTokLoc(),
"fragment can't be larger than a bundle size");
newFragment();
- CF->getParent()->ensureMinAlignment(
- Align(getAssembler().getBundleAlignSize()));
+ CF->getParent()->ensureMinAlignment(getAssembler().getBundleAlign());
}
void MCELFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *Sym,
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index bcc0bb886bc5f..5f8e177d16d68 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -490,7 +490,7 @@ void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
return;
}
PendingBA = OS.newSpecialFragment<MCBoundaryAlignFragment>(
- Align(Asm->getBundleAlignSize()), STI);
+ Asm->getBundleAlign(), STI);
// We can set LastFragment now, before the instruction is emitted, as bundling
// emits one fragment per instruction. Deferring setLastFragment to
// post-emitInstruction would risk capturing a fragment that a subsequent
@@ -514,7 +514,7 @@ void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
assert(PendingBA && "MCBoundaryAlignFragment is expected for every "
"instruction if it is not bundle-locked");
- CF->getParent()->ensureMinAlignment(Align(Asm->getBundleAlignSize()));
+ CF->getParent()->ensureMinAlignment(Asm->getBundleAlign());
// Update ReuseBA for the next BeginBundle.
ReuseBA = isPrefix(PrevInstOpcode, *MCII);
@@ -806,9 +806,8 @@ bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFragment &,
if (Asm->isBundlingEnabled() && Resolved) {
// This ensures remaining short branches have sufficient headroom to survive
// any intra-bundle shift caused by prefix padding in dividePadInBundle.
- auto BundleAlignSize = Asm->getBundleAlignSize();
- return (!isInt<8>(Value + BundleAlignSize) ||
- !isInt<8>(Value - BundleAlignSize)) ||
+ uint64_t BundleSize = Asm->getBundleAlign().value();
+ return (!isInt<8>(Value + BundleSize) || !isInt<8>(Value - BundleSize)) ||
Target.getSpecifier();
}
// If resolved, relax if the value is too big for a (signed) i8.
@@ -965,7 +964,7 @@ bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
unsigned StartOffset = Asm.getFragmentOffset(*LastF);
unsigned EndOffset = StartOffset + RemainingSize;
- auto BoundaryAlignment = Align(Asm.getBundleAlignSize());
+ Align BoundaryAlignment = Asm.getBundleAlign();
bool CrossBoundary = (StartOffset >> Log2(BoundaryAlignment)) !=
((EndOffset - 1) >> Log2(BoundaryAlignment));
@@ -979,7 +978,7 @@ bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
// such may cause fixup errors because instructions can shift by more than
// a bundle-size and labels may become unreachable. Until we come up with a
// better logic, we limits the optimization scope to a single bundle.
- RemainingSize -= EndOffset % Asm.getBundleAlignSize();
+ RemainingSize -= EndOffset % BoundaryAlignment.value();
}
assert(RemainingSize > 0);
@@ -1014,7 +1013,7 @@ bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
}
};
- unsigned TailSize = EndOffset % Asm.getBundleAlignSize();
+ unsigned TailSize = EndOffset % BoundaryAlignment.value();
if (!CrossBoundary && RemainingSize > 0 && TailSize != 0) {
padInstsForward(RemainingSize);
} else if (CrossBoundary && TailSize > 0) {
@@ -1047,7 +1046,7 @@ bool X86AsmBackend::optimizeBundleNops(const MCAssembler &Asm) const {
}
}
- if (Asm.getFragmentOffset(F) % Asm.getBundleAlignSize() == 0)
+ if (isAligned(Asm.getBundleAlign(), Asm.getFragmentOffset(F)))
Bundle.clear(); // start a new bundle
Bundle.push_back(&F);
}
>From ecf2e9a15e28d59a05b139f3e8636417ffb4c87c Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Thu, 23 Jul 2026 11:34:50 -0700
Subject: [PATCH 17/24] Extract writeControlledNops loop content to helper
---
llvm/lib/MC/MCAssembler.cpp | 24 ++++++++++++++----------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 07aa19587619e..5d3633fd07382 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -396,6 +396,18 @@ void MCAssembler::addRelocDirective(RelocDirective RD) {
relocDirectives.push_back(RD);
}
+/// Largest nop that fits in the remaining bytes without crossing a bundle
+/// boundary at Offset.
+static uint64_t maxNopBytesAt(const MCAssembler &Asm, uint64_t Remaining,
+ uint64_t MaxNopSize, uint64_t Offset) {
+ uint64_t Bytes = std::min(Remaining, MaxNopSize);
+ if (Asm.isBundlingEnabled()) {
+ uint64_t BundleSize = Asm.getBundleAlign().value();
+ Bytes = std::min(Bytes, BundleSize - (Offset & (BundleSize - 1)));
+ }
+ return Bytes;
+}
+
/// Write NOPs while limiting the maximum NOP size.
static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
uint64_t NumBytes, uint64_t FragmentOffset,
@@ -404,16 +416,8 @@ static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
uint64_t NumBytesToEmit = 0;
for (uint64_t NumBytesEmitted = 0; NumBytesEmitted < NumBytes;
NumBytesEmitted += NumBytesToEmit) {
- NumBytesToEmit = std::min(NumBytes - NumBytesEmitted, MaxNopSize);
-
- if (Asm.isBundlingEnabled()) {
- uint64_t BundleSize = Asm.getBundleAlign().value();
- uint64_t OffsetInBundle =
- (FragmentOffset + NumBytesEmitted) & (BundleSize - 1);
- uint64_t SpaceInBundle = BundleSize - OffsetInBundle;
- NumBytesToEmit = std::min(NumBytesToEmit, SpaceInBundle);
- }
-
+ NumBytesToEmit = maxNopBytesAt(Asm, NumBytes - NumBytesEmitted, MaxNopSize,
+ FragmentOffset + NumBytesEmitted);
assert(NumBytesToEmit && "try to emit zero-sized NOP");
if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit, STI)) {
>From eda522babf7082ffd23e172f0e150a5bf2fbefc3 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Fri, 24 Jul 2026 00:02:33 -0700
Subject: [PATCH 18/24] Initialize ReuseBA to false and switch IsBundeLocked to
bitfield
---
llvm/include/llvm/MC/MCSection.h | 8 ++++----
llvm/lib/MC/MCSection.cpp | 2 +-
llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 2 +-
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 7a742de3837f6..7700e4d8e04f4 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -617,10 +617,6 @@ class LLVM_ABI MCSection {
// fragment may not be fully resolved.
unsigned FirstLinkerRelaxable = -1u;
- /// If bundle-locked, we ensure all instructions in the section are placed in
- /// the same bundle.
- bool IsBundleLocked = false;
-
/// Whether this section has had instructions emitted into it.
bool HasInstructions : 1;
@@ -629,6 +625,10 @@ class LLVM_ABI MCSection {
bool IsText : 1;
bool IsBss : 1;
+ /// If bundle-locked, we ensure all instructions in the section are placed in
+ /// the same bundle.
+ bool IsBundleLocked : 1;
+
MCFragment DummyFragment;
// Mapping from subsection number to fragment list. At layout time, the
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index 2631e1e01dee6..404e0bf86d149 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -20,7 +20,7 @@ using namespace llvm;
MCSection::MCSection(StringRef Name, bool IsText, bool IsBss, MCSymbol *Begin)
: Begin(Begin), HasInstructions(false), IsRegistered(false), IsText(IsText),
- IsBss(IsBss), Name(Name) {
+ IsBss(IsBss), IsBundleLocked(false), Name(Name) {
DummyFragment.setParent(this);
}
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 5f8e177d16d68..e3d789f8e6a8e 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -124,7 +124,7 @@ class X86AsmBackend : public MCAsmBackend {
Align AlignBoundary;
unsigned TargetPrefixMax = 0;
- bool ReuseBA;
+ bool ReuseBA = false;
MCInst PrevInst;
unsigned PrevInstOpcode = 0;
MCBoundaryAlignFragment *PendingBA = nullptr;
>From 97b16e197f440f241bc830c86db3c7e62d40b84f Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Fri, 24 Jul 2026 14:07:07 -0700
Subject: [PATCH 19/24] Mark computeBoundaryAlignSize as const
---
llvm/include/llvm/MC/MCAssembler.h | 2 +-
llvm/lib/MC/MCAssembler.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index cf8612343aa74..55b340446d756 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -127,7 +127,7 @@ class MCAssembler {
void relaxSFrameFragment(MCFragment &DF);
/// Compute the padding size to boundary-align its connected fragments.
- uint64_t computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF);
+ uint64_t computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) const;
public:
/// Construct a new assembler instance.
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 5d3633fd07382..ef7c0198fde0a 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -1006,7 +1006,7 @@ static bool needPadding(uint64_t StartAddr, uint64_t Size,
}
uint64_t
-MCAssembler::computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) {
+MCAssembler::computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) const {
if (!BF.getLastFragment())
return 0;
>From 9d58df8a6420689a539d60aac820b56915e60825 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Fri, 24 Jul 2026 14:23:30 -0700
Subject: [PATCH 20/24] Simplify nop emission to go through writeControlledNops
---
llvm/lib/MC/MCAssembler.cpp | 27 +++++++--------------------
1 file changed, 7 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index ef7c0198fde0a..d55077c0be7b2 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -408,7 +408,8 @@ static uint64_t maxNopBytesAt(const MCAssembler &Asm, uint64_t Remaining,
return Bytes;
}
-/// Write NOPs while limiting the maximum NOP size.
+/// Write NumBytes of NOPs in chunks of at most MaxNopSize bytes. When bundling
+/// is enabled, no chunk crosses a bundle boundary.
static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
uint64_t NumBytes, uint64_t FragmentOffset,
uint64_t MaxNopSize,
@@ -474,16 +475,9 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
// In the nops mode, call the backend hook to write `Count` nops.
if (F.hasAlignEmitNops()) {
- if (!Asm.isBundlingEnabled()) {
- if (!Asm.getBackend().writeNopData(OS, Count, F.getSubtargetInfo()))
- reportFatalInternalError("unable to write nop sequence of " +
- Twine(Count) + " bytes");
- } else {
- // Ensure that no nop of the fill crosses a bundle boundary.
- writeControlledNops(OS, Asm, Count,
- Asm.getFragmentOffset(F) + F.getFixedSize(), Count,
- F.getSubtargetInfo());
- }
+ writeControlledNops(OS, Asm, Count,
+ Asm.getFragmentOffset(F) + F.getFixedSize(), Count,
+ F.getSubtargetInfo());
} else {
// Otherwise, write out in multiples of the value size.
for (uint64_t i = 0; i != Count; ++i) {
@@ -591,15 +585,8 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
case MCFragment::FT_BoundaryAlign: {
const MCBoundaryAlignFragment &BF = cast<MCBoundaryAlignFragment>(F);
- if (!Asm.isBundlingEnabled()) {
- if (!Asm.getBackend().writeNopData(OS, FragmentSize,
- BF.getSubtargetInfo()))
- reportFatalInternalError("unable to write nop sequence of " +
- Twine(FragmentSize) + " bytes");
- } else {
- writeControlledNops(OS, Asm, FragmentSize, Asm.getFragmentOffset(BF),
- FragmentSize, BF.getSubtargetInfo());
- }
+ writeControlledNops(OS, Asm, FragmentSize, Asm.getFragmentOffset(BF),
+ FragmentSize, BF.getSubtargetInfo());
break;
}
>From 4c01fcbd7ba3fbbef23f216a7d9b781fc0d2c851 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Mon, 27 Jul 2026 13:54:04 -0700
Subject: [PATCH 21/24] Remove unnecessary ReuseBA variable
---
.../lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index e3d789f8e6a8e..45764e90f24e7 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -124,7 +124,6 @@ class X86AsmBackend : public MCAsmBackend {
Align AlignBoundary;
unsigned TargetPrefixMax = 0;
- bool ReuseBA = false;
MCInst PrevInst;
unsigned PrevInstOpcode = 0;
MCBoundaryAlignFragment *PendingBA = nullptr;
@@ -482,9 +481,10 @@ void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
if (OS.getCurrentSectionOnly()->isBundleLocked())
return;
- // when there is a prefix MCInst not locked, we need an implicit lock between
- // the prefix and the next MCInst.
- if (ReuseBA && PendingBA &&
+ // A pending BoundaryAlign here means the previous MCInst was just a prefix,
+ // which needs an implicit lock with this one, so reuse the fragment and let
+ // this instruction extend its range.
+ if (PendingBA &&
PendingBA->getLastFragment()->getParent() == OS.getCurrentSectionOnly()) {
PendingBA->setLastFragment(OS.getCurrentFragment());
return;
@@ -516,11 +516,8 @@ void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
CF->getParent()->ensureMinAlignment(Asm->getBundleAlign());
- // Update ReuseBA for the next BeginBundle.
- ReuseBA = isPrefix(PrevInstOpcode, *MCII);
- if (ReuseBA)
- return;
- PendingBA = nullptr;
+ if (!isPrefix(PrevInstOpcode, *MCII))
+ PendingBA = nullptr;
}
/// Insert BoundaryAlignFragment before instructions to align branches.
>From dc6641c04f4eeb8135346cc8791bab0e9b781546 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Mon, 27 Jul 2026 14:37:04 -0700
Subject: [PATCH 22/24] Convert computeBoundaryAlignSize to static helper
---
llvm/include/llvm/MC/MCAssembler.h | 3 ---
llvm/include/llvm/MC/MCELFStreamer.h | 3 ---
llvm/lib/MC/MCAssembler.cpp | 17 +++++++++--------
llvm/lib/MC/MCELFStreamer.cpp | 4 ++--
4 files changed, 11 insertions(+), 16 deletions(-)
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 55b340446d756..41d082765a468 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -126,9 +126,6 @@ class MCAssembler {
void relaxDwarfCallFrameFragment(MCFragment &F);
void relaxSFrameFragment(MCFragment &DF);
- /// Compute the padding size to boundary-align its connected fragments.
- uint64_t computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) const;
-
public:
/// Construct a new assembler instance.
//
diff --git a/llvm/include/llvm/MC/MCELFStreamer.h b/llvm/include/llvm/MC/MCELFStreamer.h
index 5d276d98afff9..acd3570016265 100644
--- a/llvm/include/llvm/MC/MCELFStreamer.h
+++ b/llvm/include/llvm/MC/MCELFStreamer.h
@@ -72,9 +72,6 @@ class LLVM_ABI MCELFStreamer : public MCObjectStreamer {
void emitBundleAlignMode(Align Alignment) override;
void emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) override;
void emitBundleUnlock(const MCSubtargetInfo &STI) override;
- bool isBundleLocked() const {
- return getCurrentSectionOnly()->isBundleLocked();
- }
// This is final. Override MCTargetStreamer::finish instead for
// target-specific code.
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index d55077c0be7b2..69c2d4f20dfef 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -992,22 +992,23 @@ static bool needPadding(uint64_t StartAddr, uint64_t Size,
isAgainstBoundary(StartAddr, Size, BoundaryAlignment);
}
-uint64_t
-MCAssembler::computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) const {
- if (!BF.getLastFragment())
- return 0;
+/// Compute the padding size to boundary-align the fragments BF is responsible
+/// for.
+static uint64_t computeBoundaryAlignSize(const MCAssembler &Asm,
+ const MCBoundaryAlignFragment &BF) {
+ assert(BF.getLastFragment() && "the fragment range to align must be known");
- uint64_t AlignedOffset = getFragmentOffset(BF);
+ uint64_t AlignedOffset = Asm.getFragmentOffset(BF);
uint64_t AlignedSize = 0;
for (const MCFragment *F = BF.getNext();; F = F->getNext()) {
- AlignedSize += computeFragmentSize(*F);
+ AlignedSize += Asm.computeFragmentSize(*F);
if (F == BF.getLastFragment())
break;
}
Align BoundaryAlignment = BF.getAlignment();
- if (!isBundlingEnabled())
+ if (!Asm.isBundlingEnabled())
return needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
? offsetToAlignment(AlignedOffset, BoundaryAlignment)
: 0U;
@@ -1026,7 +1027,7 @@ void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
if (!BF.getLastFragment())
return;
- uint64_t NewSize = computeBoundaryAlignSize(BF);
+ uint64_t NewSize = computeBoundaryAlignSize(*this, BF);
if (NewSize == BF.getSize())
return;
BF.setSize(NewSize);
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 589d3aae9b57f..878e88b116bdd 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -81,7 +81,7 @@ void MCELFStreamer::emitLabelAtPos(MCSymbol *S, SMLoc Loc, MCFragment &F,
void MCELFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
MCAssembler &Asm = getAssembler();
MCFragment *CF = getCurrentFragment();
- if (Asm.isBundlingEnabled() && isBundleLocked()) {
+ if (Asm.isBundlingEnabled() && CF->getParent()->isBundleLocked()) {
getContext().reportError(getStartTokLoc(),
"unterminated .bundle_lock when changing a "
"section");
@@ -379,7 +379,7 @@ void MCELFStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
getStartTokLoc(), ".bundle_unlock forbidden when bundling is disabled");
return;
}
- if (!isBundleLocked()) {
+ if (!Sec.isBundleLocked()) {
getContext().reportError(getStartTokLoc(),
".bundle_unlock without matching lock");
return;
>From 3b515247de5c5485802e1c8781f8684e347c6851 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Fri, 31 Jul 2026 20:56:31 -0700
Subject: [PATCH 23/24] [MC][X86] Fix bundle boundary violations
Fold a MCBoundaryAlignFragment's padding only into instructions sharing its
bundle. optimizeBundleNops starts a new bundle at fragments beginning on a
boundary and treats unlisted fragment kinds as transparent, so a fragment that
spans a boundary (.nops, .space) lets --x86-pad-max-prefix-size grow an
instruction in an earlier bundle or inflate a .bundle_lock group past the
bundle size. Track the bundle by offset, clear the candidates on any fragment
whose size depends on its offset, and cap what a group absorbs at the slack in
its bundle. The cap subsumes the CrossBoundary split: the backward budget is
the padding within its own bundle, the forward budget BundleSize - GroupSize
for align_to_end and zero otherwise.
Require the pending boundary fragment to be adjacent before an instruction
extends it. A prefix followed by .bundle_lock otherwise leaves a stale fragment
spanning the group, and the instruction after .bundle_unlock gets none.
Diagnose the mirror case, a group ending in a bare prefix.
Reject .bundle_align_mode where nothing enforces the boundaries: non-ELF
formats and ELF targets without a bundling backend, which today pad
.bundle_lock groups while letting instructions straddle boundaries. Reject
.bundle_lock outside an executable section, where it writes nops into data, and
diagnose one left open at end of file.
Move the lock flag from MCSection to MCObjectStreamer; a section cannot be
switched while a group is open. Clear the bundling state in reset(), which
frees the fragments PendingBA points into. Restrict the relaxation headroom to
relaxable branches, as immediates do not shift. Reject `.bundle_align_mode 0`
in the parser's 1-30 range check instead of separately in the streamer.
---
llvm/docs/AlignedBundling.rst | 21 +-
llvm/include/llvm/MC/MCAsmBackend.h | 4 +
llvm/include/llvm/MC/MCObjectStreamer.h | 7 +
llvm/include/llvm/MC/MCSection.h | 7 -
llvm/include/llvm/MC/MCStreamer.h | 4 +-
llvm/lib/MC/MCAssembler.cpp | 42 ++--
llvm/lib/MC/MCELFStreamer.cpp | 110 ++++-----
llvm/lib/MC/MCObjectStreamer.cpp | 7 +-
llvm/lib/MC/MCParser/AsmParser.cpp | 7 +-
llvm/lib/MC/MCSection.cpp | 2 +-
llvm/lib/MC/MCStreamer.cpp | 17 +-
.../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 231 +++++++++---------
.../MC/X86/AlignedBundling/bundle-errors.s | 70 +++++-
.../test/MC/X86/AlignedBundling/bundle-lock.s | 16 ++
.../MC/X86/AlignedBundling/prefix-padding.s | 41 +++-
llvm/test/MC/X86/AlignedBundling/section.s | 6 +-
16 files changed, 354 insertions(+), 238 deletions(-)
diff --git a/llvm/docs/AlignedBundling.rst b/llvm/docs/AlignedBundling.rst
index 23100ad7d6add..a7b47f936ef97 100644
--- a/llvm/docs/AlignedBundling.rst
+++ b/llvm/docs/AlignedBundling.rst
@@ -24,6 +24,8 @@ Isolation (LFI) <LFI>`.
.. note::
The current LLVM implementation supports bundling only for x86 ELF targets.
+ Other targets and object file formats reject the directives below rather
+ than emit unbundled code silently.
``.bundle_align_mode``
======================
@@ -33,9 +35,10 @@ Isolation (LFI) <LFI>`.
.bundle_align_mode abs-expr
Enables aligned bundle mode and sets the bundle size to ``2^abs-expr`` bytes,
-where ``abs-expr`` is a power-of-two exponent between 0 and 30 (as for the
+where ``abs-expr`` is a power-of-two exponent between 1 and 30 (as for the
``.p2align`` directive). For example, ``.bundle_align_mode 5`` selects 32-byte
-bundles.
+bundles. Unlike GNU as, ``.bundle_align_mode 0`` is rejected: bundling cannot be
+turned off once enabled.
While bundling is enabled, the assembler ensures that no single instruction
spans a boundary between two bundles. When an instruction would not fit in the
@@ -65,10 +68,11 @@ within one bundle rather than straddling a boundary.
The enclosed sequence must fit within a single bundle -- it is an error if the
total size of the locked instructions exceeds the bundle size.
-Both directives are only valid after bundle mode has been enabled with
-``.bundle_align_mode``. A ``.bundle_unlock`` must be matched by a preceding
-``.bundle_lock`` in the same section, and a section may not be switched while a
-``.bundle_lock`` is open.
+Both directives are only valid in an executable section after bundle mode has
+been enabled with ``.bundle_align_mode``. A ``.bundle_unlock`` must be matched
+by a preceding ``.bundle_lock``, and a section may not be switched while a
+``.bundle_lock`` is open. A group may not end in a bare instruction prefix,
+which the assembler could not keep attached to the instruction it modifies.
``align_to_end``
----------------
@@ -95,3 +99,8 @@ to them, avoiding standalone no-ops. This is controlled by:
Maximum number of prefixes the assembler may add to an instruction for
padding. ``0`` (the default) disables prefix padding, so only no-op
instructions are used.
+
+Padding is only ever traded for instruction bytes within the bundle that holds
+it, so this cannot move an instruction or a locked group across a boundary. It
+does change where labels land, so it may break assumptions about labels
+corresponding to particular instructions.
diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index c91cd772669f5..816815ab928f8 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -60,6 +60,7 @@ class LLVM_ABI MCAsmBackend {
bool AllowAutoPadding = false;
bool AllowEnhancedRelaxation = false;
+ bool AllowBundling = false;
public:
MCAsmBackend(const MCAsmBackend &) = delete;
@@ -79,6 +80,9 @@ class LLVM_ABI MCAsmBackend {
/// emitted into RelaxableFragment and then we can increase its size in a
/// tricky way for optimization.
bool allowEnhancedRelaxation() const { return AllowEnhancedRelaxation; }
+ /// Return true if this target implements `.bundle_align_mode`. Other targets
+ /// reject the directive instead of emitting unbundled code.
+ bool allowBundling() const { return AllowBundling; }
/// lifetime management
virtual void reset() {}
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index 003f6106d15ba..7f5875c90d215 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -66,6 +66,10 @@ class LLVM_ABI MCObjectStreamer : public MCStreamer {
void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
protected:
+ // Within .bundle_lock/.bundle_unlock. A section cannot be switched while a
+ // group is open, so no per-section state is needed.
+ bool BundleLocked = false;
+
MCObjectStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> TAB,
std::unique_ptr<MCObjectWriter> OW,
std::unique_ptr<MCCodeEmitter> Emitter);
@@ -88,6 +92,9 @@ class LLVM_ABI MCObjectStreamer : public MCStreamer {
MCAssembler &getAssembler() { return *Assembler; }
MCAssembler *getAssemblerPtr() override;
+
+ bool isBundleLocked() const { return BundleLocked; }
+
/// \name MCStreamer Interface
/// @{
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 7700e4d8e04f4..3574aec9731c2 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -625,10 +625,6 @@ class LLVM_ABI MCSection {
bool IsText : 1;
bool IsBss : 1;
- /// If bundle-locked, we ensure all instructions in the section are placed in
- /// the same bundle.
- bool IsBundleLocked : 1;
-
MCFragment DummyFragment;
// Mapping from subsection number to fragment list. At layout time, the
@@ -687,9 +683,6 @@ class LLVM_ABI MCSection {
bool isLinkerRelaxable() const { return FirstLinkerRelaxable != -1u; }
void setFirstLinkerRelaxable(unsigned Order) { FirstLinkerRelaxable = Order; }
- bool isBundleLocked() const { return IsBundleLocked; }
- void setIsBundleLocked(bool Value) { IsBundleLocked = Value; }
-
MCFragment &getDummyFragment() { return DummyFragment; }
FragList *curFragList() const { return CurFragList; }
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index d333bf4f9774d..a6b24a5a9a18d 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -1133,7 +1133,9 @@ class LLVM_ABI MCStreamer {
/// Enable aligned instruction bundling with the given bundle size, from
/// this point onward. Once enabled, bundling cannot be disabled and the
- /// bundle size cannot be changed.
+ /// bundle size cannot be changed. \p Alignment must be within [2, 2^30].
+ /// The default implementations report an error: silently emitting unbundled
+ /// code would defeat the sandboxing schemes bundling exists for.
virtual void emitBundleAlignMode(Align Alignment);
/// The following instructions are a bundle-locked group.
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 69c2d4f20dfef..fcdcdbc96426f 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -92,6 +92,7 @@ void MCAssembler::reset() {
HasLayout = false;
HasFinalLayout = false;
RelaxAll = false;
+ BundleAlign.reset();
Sections.clear();
Symbols.clear();
ThumbFuncs.clear();
@@ -396,37 +397,24 @@ void MCAssembler::addRelocDirective(RelocDirective RD) {
relocDirectives.push_back(RD);
}
-/// Largest nop that fits in the remaining bytes without crossing a bundle
-/// boundary at Offset.
-static uint64_t maxNopBytesAt(const MCAssembler &Asm, uint64_t Remaining,
- uint64_t MaxNopSize, uint64_t Offset) {
- uint64_t Bytes = std::min(Remaining, MaxNopSize);
- if (Asm.isBundlingEnabled()) {
- uint64_t BundleSize = Asm.getBundleAlign().value();
- Bytes = std::min(Bytes, BundleSize - (Offset & (BundleSize - 1)));
- }
- return Bytes;
-}
-
-/// Write NumBytes of NOPs in chunks of at most MaxNopSize bytes. When bundling
-/// is enabled, no chunk crosses a bundle boundary.
+/// Write \p NumBytes of NOPs at \p Offset in chunks of at most \p MaxNopSize.
+/// When bundling is enabled, no chunk crosses a bundle boundary.
static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
- uint64_t NumBytes, uint64_t FragmentOffset,
+ uint64_t NumBytes, uint64_t Offset,
uint64_t MaxNopSize,
const MCSubtargetInfo *STI) {
- uint64_t NumBytesToEmit = 0;
- for (uint64_t NumBytesEmitted = 0; NumBytesEmitted < NumBytes;
- NumBytesEmitted += NumBytesToEmit) {
- NumBytesToEmit = maxNopBytesAt(Asm, NumBytes - NumBytesEmitted, MaxNopSize,
- FragmentOffset + NumBytesEmitted);
- assert(NumBytesToEmit && "try to emit zero-sized NOP");
-
- if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit, STI)) {
- reportFatalInternalError(
- "unable to write NOP sequence of the remaining " +
- Twine(NumBytesToEmit) + " bytes");
- return;
+ while (NumBytes) {
+ uint64_t Size = std::min(NumBytes, MaxNopSize);
+ if (Asm.isBundlingEnabled()) {
+ uint64_t BundleSize = Asm.getBundleAlign().value();
+ Size = std::min(Size, BundleSize - (Offset & (BundleSize - 1)));
}
+ assert(Size && "try to emit zero-sized NOP");
+ if (!Asm.getBackend().writeNopData(OS, Size, STI))
+ reportFatalInternalError("unable to write nop sequence of " +
+ Twine(Size) + " bytes");
+ NumBytes -= Size;
+ Offset += Size;
}
}
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 878e88b116bdd..03bbf4e81120e 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -80,13 +80,11 @@ void MCELFStreamer::emitLabelAtPos(MCSymbol *S, SMLoc Loc, MCFragment &F,
void MCELFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
MCAssembler &Asm = getAssembler();
- MCFragment *CF = getCurrentFragment();
- if (Asm.isBundlingEnabled() && CF->getParent()->isBundleLocked()) {
- getContext().reportError(getStartTokLoc(),
- "unterminated .bundle_lock when changing a "
- "section");
+ if (isBundleLocked()) {
+ getContext().reportError(
+ getStartTokLoc(), "unterminated .bundle_lock when changing a section");
// Clean up bundle state to allow continuing.
- CF->getParent()->setIsBundleLocked(false);
+ BundleLocked = false;
BundleBA = nullptr;
}
auto *SectionELF = static_cast<const MCSectionELF *>(Section);
@@ -326,92 +324,85 @@ void MCELFStreamer::emitIdent(StringRef IdentString) {
}
void MCELFStreamer::emitBundleAlignMode(Align Alignment) {
- if (Log2(Alignment) > 30)
- return getContext().reportError(
- getStartTokLoc(), ".bundle_align_mode alignment must be <= 30");
- if (Alignment == 1)
+ MCAssembler &Asm = getAssembler();
+ SMLoc Loc = getStartTokLoc();
+
+ if (!Asm.getBackend().allowBundling())
return getContext().reportError(
- getStartTokLoc(), "disabling .bundle_align_mode is not supported");
- MCAssembler &Assembler = getAssembler();
- if (Assembler.isBundlingEnabled() &&
- Assembler.getBundleAlign() != Alignment) {
- getContext().reportError(getStartTokLoc(),
- ".bundle_align_mode cannot be changed once set");
+ Loc, "aligned bundling is not supported by this target");
+ if (Asm.isBundlingEnabled()) {
+ if (Asm.getBundleAlign() != Alignment)
+ getContext().reportError(Loc,
+ ".bundle_align_mode cannot be changed once set");
return;
}
- if (!Assembler.isBundlingEnabled() &&
- Assembler.getBackend().allowAutoPadding())
+ // Enable bundling even after the error, to avoid cascading diagnostics from
+ // later .bundle_lock directives.
+ if (Asm.getBackend().allowAutoPadding())
getContext().reportError(
- getStartTokLoc(),
- ".bundle_align_mode is incompatible with branch alignment");
+ Loc, ".bundle_align_mode is incompatible with branch alignment");
+
setAllowAutoPadding(true);
- Assembler.setBundleAlign(Alignment);
+ Asm.setBundleAlign(Alignment);
}
void MCELFStreamer::emitBundleLock(bool AlignToEnd,
const MCSubtargetInfo &STI) {
- MCSection &Sec = *getCurrentSectionOnly();
- auto &Asm = getAssembler();
-
- if (!Asm.isBundlingEnabled()) {
- getContext().reportError(
- getStartTokLoc(), ".bundle_lock forbidden when bundling is disabled");
- return;
- }
+ MCAssembler &Asm = getAssembler();
+ SMLoc Loc = getStartTokLoc();
- if (Sec.isBundleLocked()) {
- getContext().reportError(getStartTokLoc(),
- "nested .bundle_lock is not allowed");
- return;
- }
- Sec.setIsBundleLocked(true);
+ if (!Asm.isBundlingEnabled())
+ return getContext().reportError(
+ Loc, ".bundle_lock forbidden when bundling is disabled");
+ if (isBundleLocked())
+ return getContext().reportError(Loc, "nested .bundle_lock is not allowed");
+ // Padding a group is only meaningful where nops are instructions.
+ if (!getCurrentSectionOnly()->isText())
+ return getContext().reportError(
+ Loc, ".bundle_lock is only allowed in an executable section");
+ BundleLocked = true;
BundleBA =
newSpecialFragment<MCBoundaryAlignFragment>(Asm.getBundleAlign(), STI);
BundleBA->setAlignToEnd(AlignToEnd);
}
void MCELFStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
- MCSection &Sec = *getCurrentSectionOnly();
-
- if (!getAssembler().isBundlingEnabled()) {
- getContext().reportError(
- getStartTokLoc(), ".bundle_unlock forbidden when bundling is disabled");
- return;
- }
- if (!Sec.isBundleLocked()) {
- getContext().reportError(getStartTokLoc(),
- ".bundle_unlock without matching lock");
- return;
- }
+ MCAssembler &Asm = getAssembler();
+ SMLoc Loc = getStartTokLoc();
- Sec.setIsBundleLocked(false);
+ if (!Asm.isBundlingEnabled())
+ return getContext().reportError(
+ Loc, ".bundle_unlock forbidden when bundling is disabled");
+ if (!isBundleLocked())
+ return getContext().reportError(Loc,
+ ".bundle_unlock without matching lock");
+ BundleLocked = false;
MCFragment *CF = getCurrentFragment();
BundleBA->setLastFragment(CF);
- // Bundle overflow check.
- uint64_t AlignedSize = 0;
+
+ uint64_t GroupSize = 0;
for (const MCFragment *F = BundleBA->getNext();; F = F->getNext()) {
if (F->getKind() == MCFragment::FT_Align ||
F->getKind() == MCFragment::FT_Org) {
- getContext().reportError(getStartTokLoc(),
- "alignment and .org directives are not "
- "supported inside a .bundle_lock group");
+ getContext().reportError(Loc, "alignment and .org directives are not "
+ "supported inside a .bundle_lock group");
break;
}
- AlignedSize += getAssembler().computeFragmentSize(*F);
+ GroupSize += Asm.computeFragmentSize(*F);
if (F == BundleBA->getLastFragment())
break;
}
BundleBA = nullptr;
- if (AlignedSize > getAssembler().getBundleAlign().value())
- getContext().reportError(getStartTokLoc(),
- "fragment can't be larger than a bundle size");
+ if (GroupSize > Asm.getBundleAlign().value())
+ getContext().reportError(
+ Loc, ".bundle_lock group is larger than the bundle size");
newFragment();
- CF->getParent()->ensureMinAlignment(getAssembler().getBundleAlign());
+ CF->getParent()->ensureMinAlignment(Asm.getBundleAlign());
}
void MCELFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *Sym,
@@ -456,6 +447,9 @@ void MCELFStreamer::finalizeCGProfile() {
}
void MCELFStreamer::finishImpl() {
+ if (isBundleLocked())
+ getContext().reportError(getStartTokLoc(), "unterminated .bundle_lock");
+
// Emit .note.GNU-stack, similar to AsmPrinter::doFinalization.
MCContext &Ctx = getContext();
auto *StackSec = Ctx.getAsmInfo().getStackSection(Ctx,
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index e5b36c30e8b18..7a64fbbd66712 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -175,6 +175,7 @@ void MCObjectStreamer::reset() {
}
EmitEHFrame = true;
EmitDebugFrame = false;
+ BundleLocked = false;
FragStorage.clear();
FragSpace = 0;
SpecialFragAllocator.Reset();
@@ -422,10 +423,10 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
return I;
};
- // To enable better bundle-nop optimization, we emit every instruction
- // as a relaxable fragment.
+ // Bundling emits one relaxable fragment per instruction so that finishLayout
+ // can fold padding into instruction encodings.
if (Assembler.isBundlingEnabled()) {
- if (Sec->isBundleLocked() || Assembler.getRelaxAll())
+ if (BundleLocked || Assembler.getRelaxAll())
emitInstToFragment(relaxToFixpoint(Inst), STI);
else
emitInstToFragment(Inst, STI);
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 4bc7be913c2b2..040452134abe2 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -6011,13 +6011,14 @@ bool AsmParser::parseDirectiveAddrsigSym() {
/// ::= {.bundle_align_mode} expression
bool AsmParser::parseDirectiveBundleAlignMode() {
// Expect a single argument: an expression that evaluates to a constant
- // in the inclusive range 0-30.
+ // in the inclusive range 1-30. Unlike GNU as, 0 (disabling bundling) is not
+ // supported.
SMLoc ExprLoc = getLexer().getLoc();
int64_t AlignSizePow2;
if (checkForValidSection() || parseAbsoluteExpression(AlignSizePow2) ||
parseEOL() ||
- check(AlignSizePow2 < 0 || AlignSizePow2 > 30, ExprLoc,
- "invalid bundle alignment size (expected between 0 and 30)"))
+ check(AlignSizePow2 < 1 || AlignSizePow2 > 30, ExprLoc,
+ "invalid bundle alignment size (expected between 1 and 30)"))
return true;
getStreamer().emitBundleAlignMode(Align(1ULL << AlignSizePow2));
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index 404e0bf86d149..2631e1e01dee6 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -20,7 +20,7 @@ using namespace llvm;
MCSection::MCSection(StringRef Name, bool IsText, bool IsBss, MCSymbol *Begin)
: Begin(Begin), HasInstructions(false), IsRegistered(false), IsText(IsText),
- IsBss(IsBss), IsBundleLocked(false), Name(Name) {
+ IsBss(IsBss), Name(Name) {
DummyFragment.setParent(this);
}
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index e4fc9d4932ee6..90e560d845ca8 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -1527,9 +1527,20 @@ void MCStreamer::emitCodeAlignment(Align Alignment, const MCSubtargetInfo &STI,
unsigned MaxBytesToEmit) {}
void MCStreamer::emitValueToOffset(const MCExpr *Offset, unsigned char Value,
SMLoc Loc) {}
-void MCStreamer::emitBundleAlignMode(Align Alignment) {}
-void MCStreamer::emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) {}
-void MCStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {}
+static void reportBundlingUnsupported(MCStreamer &S) {
+ S.getContext().reportError(
+ S.getStartTokLoc(),
+ "aligned bundling is not supported by this object file format");
+}
+void MCStreamer::emitBundleAlignMode(Align) {
+ reportBundlingUnsupported(*this);
+}
+void MCStreamer::emitBundleLock(bool, const MCSubtargetInfo &) {
+ reportBundlingUnsupported(*this);
+}
+void MCStreamer::emitBundleUnlock(const MCSubtargetInfo &) {
+ reportBundlingUnsupported(*this);
+}
void MCStreamer::finishImpl() {}
bool MCStreamer::popSection() {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 45764e90f24e7..5d245f7f90cf3 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -126,6 +126,7 @@ class X86AsmBackend : public MCAsmBackend {
MCInst PrevInst;
unsigned PrevInstOpcode = 0;
+ bool PrefixEndsBundleLock = false;
MCBoundaryAlignFragment *PendingBA = nullptr;
std::pair<MCFragment *, size_t> PrevInstPosition;
@@ -163,6 +164,16 @@ class X86AsmBackend : public MCAsmBackend {
AlignBoundary != Align(1) && AlignBranchType != X86::AlignBranchNone;
AllowEnhancedRelaxation =
AllowAutoPadding && TargetPrefixMax != 0 && X86PadForBranchAlign;
+ AllowBundling = true;
+ }
+
+ // The streamer frees the fragments these point into.
+ void reset() override {
+ PrevInst = MCInst();
+ PrevInstOpcode = 0;
+ PrefixEndsBundleLock = false;
+ PendingBA = nullptr;
+ PrevInstPosition = {};
}
void emitInstructionBegin(MCObjectStreamer &OS, const MCInst &Inst,
@@ -202,8 +213,8 @@ class X86AsmBackend : public MCAsmBackend {
bool padInstsBackward(SmallVectorImpl<MCFragment *> &Relaxable,
unsigned &RemainingSize) const;
- bool dividePadInBundle(const MCAssembler &Asm,
- ArrayRef<MCFragment *> Peephole) const;
+ bool foldBundlePad(const MCAssembler &Asm, MCBoundaryAlignFragment &BF,
+ SmallVectorImpl<MCFragment *> &Relaxable) const;
bool optimizeBundleNops(const MCAssembler &Asm) const;
unsigned getMaximumNopSize(const MCSubtargetInfo &STI) const override;
@@ -472,20 +483,26 @@ void X86_MC::emitInstruction(MCObjectStreamer &S, const MCInst &Inst,
Backend.emitInstructionEnd(S, Inst);
}
-/// If the upcoming instruction is inside the bundle lock, do nothing so that
-/// the ObjectStreamer emits the instruction to the current fragment. If not, it
-/// creates a new BA to group bundled fragments. If PrevInst is just a prefix,
-/// we can reuse old BA.
+/// Open a MCBoundaryAlignFragment for the upcoming instruction so that layout
+/// can pad it into the next bundle. Within .bundle_lock the group's fragment
+/// already covers it.
void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
assert(Asm->isBundlingEnabled());
- if (OS.getCurrentSectionOnly()->isBundleLocked())
+ // The prefix stays in the group while this instruction gets its own
+ // fragment, so padding may land between the two.
+ if (PrefixEndsBundleLock && OS.getCurrentFragment() != PrevInstPosition.first)
+ getContext().reportError(OS.getStartTokLoc(),
+ "instruction prefix cannot be the last "
+ "instruction of a .bundle_lock group");
+
+ if (OS.isBundleLocked())
return;
- // A pending BoundaryAlign here means the previous MCInst was just a prefix,
- // which needs an implicit lock with this one, so reuse the fragment and let
- // this instruction extend its range.
+ // A pending fragment means the previous MCInst was a prefix, which must stay
+ // with this one: extend its range. Adjacency rejects a fragment left stale by
+ // an intervening .bundle_lock group.
if (PendingBA &&
- PendingBA->getLastFragment()->getParent() == OS.getCurrentSectionOnly()) {
+ PendingBA->getLastFragment()->getNext() == OS.getCurrentFragment()) {
PendingBA->setLastFragment(OS.getCurrentFragment());
return;
}
@@ -499,22 +516,20 @@ void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
PendingBA->setLastFragment(OS.getCurrentFragment());
}
-/// If the just-emitted instruction is inside the bundle lock, check the current
-/// fragment is non-zero to ensure the instruction is placed as expected.
-/// emitBundleUnlock will close the fragment and start a new empty fragment. If
-/// it is not locked, finalize pending BA Fragment. If it is a prefix, let the
-/// next instruction reuse the same BA.
+/// Close the fragment opened by emitInstructionBeginBundle, unless the
+/// instruction was a prefix, in which case the next one extends it.
void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
assert(Asm->isBundlingEnabled());
- MCFragment *CF = OS.getCurrentFragment();
- // We're still inside the lock, do not close the current fragment with BA.
- if (OS.getCurrentSectionOnly()->isBundleLocked())
+ if (OS.isBundleLocked()) {
+ PrefixEndsBundleLock = isPrefix(PrevInstOpcode, *MCII);
return;
+ }
+ PrefixEndsBundleLock = false;
assert(PendingBA && "MCBoundaryAlignFragment is expected for every "
"instruction if it is not bundle-locked");
- CF->getParent()->ensureMinAlignment(Asm->getBundleAlign());
+ OS.getCurrentSectionOnly()->ensureMinAlignment(Asm->getBundleAlign());
if (!isPrefix(PrevInstOpcode, *MCII))
PendingBA = nullptr;
@@ -795,24 +810,26 @@ bool X86AsmBackend::mayNeedRelaxation(unsigned Opcode,
Operands[Operands.size() - 1 - SkipOperands].isExpr());
}
-bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFragment &,
+bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFragment &F,
const MCFixup &Fixup,
const MCValue &Target,
uint64_t Value,
bool Resolved) const {
- if (Asm->isBundlingEnabled() && Resolved) {
- // This ensures remaining short branches have sufficient headroom to survive
- // any intra-bundle shift caused by prefix padding in dividePadInBundle.
- uint64_t BundleSize = Asm->getBundleAlign().value();
- return (!isInt<8>(Value + BundleSize) || !isInt<8>(Value - BundleSize)) ||
- Target.getSpecifier();
- }
// If resolved, relax if the value is too big for a (signed) i8.
//
// Currently, `jmp local at plt` relaxes JMP even if the offset is small,
// different from gas.
- if (Resolved)
- return !isInt<8>(Value) || Target.getSpecifier();
+ if (Resolved) {
+ // finishLayout folds padding into encodings after relaxation, shifting a
+ // branch and its target within their bundles. Keep a bundle of headroom.
+ // Immediates do not shift.
+ int64_t Slack = Asm->isBundlingEnabled() && TargetPrefixMax != 0 &&
+ isRelaxableBranch(F.getOpcode())
+ ? Asm->getBundleAlign().value()
+ : 0;
+ return !isInt<8>(int64_t(Value) + Slack) ||
+ !isInt<8>(int64_t(Value) - Slack) || Target.getSpecifier();
+ }
// Otherwise, relax unless there is a @ABS8 specifier.
if (Fixup.getKind() == FK_Data_1 && Target.getAddSym() &&
@@ -948,104 +965,94 @@ bool X86AsmBackend::padInstsBackward(SmallVectorImpl<MCFragment *> &Relaxable,
return Changed;
}
-// Peephole is a list of Fragments that ends with non-zero-sized
-// BoundaryAlignFragment. Most of the time it will be every instruction within a
-// bundle, but there can be a partial bundle if it has nops in the middle(e.g.,
-// align_to_end).
-bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
- ArrayRef<MCFragment *> Peephole) const {
- bool Changed = false;
- auto *LastF = Peephole.back();
- unsigned RemainingSize =
- Asm.computeFragmentSize(*LastF) - LastF->getFixedSize();
-
- unsigned StartOffset = Asm.getFragmentOffset(*LastF);
- unsigned EndOffset = StartOffset + RemainingSize;
- Align BoundaryAlignment = Asm.getBundleAlign();
- bool CrossBoundary = (StartOffset >> Log2(BoundaryAlignment)) !=
- ((EndOffset - 1) >> Log2(BoundaryAlignment));
-
- if (CrossBoundary) {
- // i.e., this pad is a mix of suffix fragment of one bundle + prefix of the
- // very next bundle. It prevents overflow of the first bundle when Peephole
- // contains more than one bundle.
- //
- // This design limits the possibly further-optimized code, which might be
- // achieved by migrating some instructions to the next bundle, but doing
- // such may cause fixup errors because instructions can shift by more than
- // a bundle-size and labels may become unreachable. Until we come up with a
- // better logic, we limits the optimization scope to a single bundle.
- RemainingSize -= EndOffset % BoundaryAlignment.value();
- }
- assert(RemainingSize > 0);
-
- SmallVector<MCFragment *, 4> Relaxable;
- for (auto *FIB : Peephole) {
- if (FIB->getKind() == MCFragment::FT_Data) // Skip and ignore
- continue;
-
- if (FIB->getKind() == MCFragment::FT_Align) {
- // p2align within a bundle
- Relaxable.clear();
- continue;
- }
-
- if (FIB->getKind() == MCFragment::FT_Relaxable) {
- Relaxable.push_back(FIB);
- continue;
- }
- }
-
- // First, try padding previous instructions.
- Changed |= padInstsBackward(Relaxable, RemainingSize);
-
- // Second, try padding following instructions.
- auto padInstsForward = [&](unsigned &Size) {
- auto *BF = cast<MCBoundaryAlignFragment>(LastF);
- for (auto *F = BF->getNext();; F = F->getNext()) {
- if (F->getKind() == MCFragment::FT_Relaxable)
- Changed |= padInstructionEncoding(*F, Asm.getEmitter(), Size);
- if (F == BF->getLastFragment() || Size == 0)
+/// Trade the padding held by \p BF for ignored prefixes on the instructions
+/// around it. Padding never leaves its own bundle, so no instruction or
+/// bundle-locked group moves across a boundary. \p Relaxable holds the
+/// preceding instructions in that bundle and is consumed.
+bool X86AsmBackend::foldBundlePad(
+ const MCAssembler &Asm, MCBoundaryAlignFragment &BF,
+ SmallVectorImpl<MCFragment *> &Relaxable) const {
+ const uint64_t BundleSize = Asm.getBundleAlign().value();
+ const uint64_t PadStart = Asm.getFragmentOffset(BF);
+ unsigned Remaining = BF.getSize();
+
+ // Only padding in PadStart's own bundle may move backward; an align_to_end
+ // group can push the rest into the next bundle.
+ unsigned Budget =
+ std::min<uint64_t>(Remaining, BundleSize - PadStart % BundleSize);
+ unsigned Left = Budget;
+ bool Changed = padInstsBackward(Relaxable, Left);
+ Remaining -= Budget - Left;
+
+ // Absorbing padding moves the group's start earlier while its end is pinned,
+ // so it may only grow into the slack in its bundle: BundleSize - GroupSize
+ // for align_to_end, zero for a group that already starts on a boundary.
+ if (BF.isAlignToEnd() && Remaining) {
+ uint64_t GroupSize = 0;
+ for (const MCFragment *F = BF.getNext();; F = F->getNext()) {
+ GroupSize += Asm.computeFragmentSize(*F);
+ if (F == BF.getLastFragment())
break;
}
- };
-
- unsigned TailSize = EndOffset % BoundaryAlignment.value();
- if (!CrossBoundary && RemainingSize > 0 && TailSize != 0) {
- padInstsForward(RemainingSize);
- } else if (CrossBoundary && TailSize > 0) {
- unsigned NextRemainingSize = TailSize;
- padInstsForward(NextRemainingSize);
- RemainingSize += NextRemainingSize;
+ if (GroupSize < BundleSize) {
+ Left = Budget = std::min<uint64_t>(Remaining, BundleSize - GroupSize);
+ for (MCFragment *F = BF.getNext(); F; F = F->getNext()) {
+ if (F->getKind() == MCFragment::FT_Relaxable)
+ Changed |= padInstructionEncoding(*F, Asm.getEmitter(), Left);
+ if (F == BF.getLastFragment() || Left == 0)
+ break;
+ }
+ Remaining -= Budget - Left;
+ }
}
- // Record the computed padding on the BoundaryAlignFragment.
- cast<MCBoundaryAlignFragment>(LastF)->setSize(RemainingSize);
-
+ BF.setSize(Remaining);
return Changed;
}
bool X86AsmBackend::optimizeBundleNops(const MCAssembler &Asm) const {
+ const uint64_t BundleSize = Asm.getBundleAlign().value();
bool Changed = false;
for (MCSection &Sec : Asm) {
if (!Sec.isText())
continue;
- SmallVector<MCFragment *, 4> Bundle;
+ // Instructions preceding the next padding and sharing its bundle.
+ SmallVector<MCFragment *, 8> Relaxable;
+ // Folding leaves stale offsets until the next layout, so skip the
+ // rewritten range.
+ const MCFragment *ResumeAfter = nullptr;
for (MCFragment &F : Sec) {
- if (F.getKind() == llvm::MCFragment::FT_BoundaryAlign) {
- unsigned RemainingSize = Asm.computeFragmentSize(F) - F.getFixedSize();
- if (RemainingSize > 0) {
- Bundle.push_back(&F);
- Changed |= dividePadInBundle(Asm, Bundle);
- Bundle.clear();
- continue;
- }
+ if (ResumeAfter) {
+ if (&F == ResumeAfter)
+ ResumeAfter = nullptr;
+ continue;
}
+ uint64_t Offset = Asm.getFragmentOffset(F);
+ if (!Relaxable.empty() &&
+ Asm.getFragmentOffset(*Relaxable.front()) / BundleSize !=
+ Offset / BundleSize)
+ Relaxable.clear();
- if (isAligned(Asm.getBundleAlign(), Asm.getFragmentOffset(F)))
- Bundle.clear(); // start a new bundle
- Bundle.push_back(&F);
+ switch (F.getKind()) {
+ case MCFragment::FT_BoundaryAlign: {
+ auto &BF = static_cast<MCBoundaryAlignFragment &>(F);
+ if (!BF.getSize())
+ break; // Nothing to fold, and not a barrier.
+ Changed |= foldBundlePad(Asm, BF, Relaxable);
+ ResumeAfter = BF.getLastFragment();
+ break;
+ }
+ case MCFragment::FT_Relaxable:
+ Relaxable.push_back(&F);
+ break;
+ case MCFragment::FT_Data:
+ break; // Fixed bytes, safe to shift.
+ default:
+ // Other kinds may change size when shifted (.p2align, .org, LEBs).
+ Relaxable.clear();
+ break;
+ }
}
}
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
index e417198783bee..c1debb720ec7c 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
@@ -7,44 +7,52 @@
# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/nested-lock.s 2>&1 | FileCheck %t/nested-lock.s
-# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/mode-zero.s 2>&1 | FileCheck %t/mode-zero.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/mode-bad-size.s 2>&1 | FileCheck %t/mode-bad-size.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/align-in-lock.s 2>&1 | FileCheck %t/align-in-lock.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/unterminated-lock.s 2>&1 | FileCheck %t/unterminated-lock.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/lock-in-data.s 2>&1 | FileCheck %t/lock-in-data.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/prefix-ends-lock.s 2>&1 | FileCheck %t/prefix-ends-lock.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 -x86-branches-within-32B-boundaries %t/bundle-with-align-branch.s 2>&1 | FileCheck %t/bundle-with-align-branch.s
# RUN: not llvm-mc -filetype=obj -triple x86_64 -x86-align-branch-boundary=32 -x86-align-branch=jmp %t/bundle-with-align-branch.s 2>&1 | FileCheck %t/bundle-with-align-branch.s
+## Formats and targets without a bundling implementation reject the directives.
+# RUN: not llvm-mc -filetype=obj -triple x86_64-apple-darwin %t/unsupported.s 2>&1 | FileCheck %t/unsupported.s --check-prefix=FORMAT
+# RUN: not llvm-mc -filetype=obj -triple x86_64-windows-msvc %t/unsupported.s 2>&1 | FileCheck %t/unsupported.s --check-prefix=FORMAT
+# RUN: %if aarch64-registered-target %{ not llvm-mc -filetype=obj -triple aarch64 %t/unsupported.s 2>&1 | FileCheck %t/unsupported.s --check-prefix=TARGET %}
+
-## .bundle_lock can't come without a .bundle_align_mode before it
#--- lock-without-mode.s
+## .bundle_lock can't come without a .bundle_align_mode before it
imull $17, %ebx, %ebp
# CHECK: [[#@LINE+1]]:3: error: .bundle_lock forbidden when bundling is disabled
.bundle_lock
# CHECK: [[#@LINE+1]]:3: error: .bundle_unlock forbidden when bundling is disabled
.bundle_unlock
-## .bundle_align_mode needs a following integer value
#--- mode-without-arg.s
+## .bundle_align_mode needs a following integer value
# CHECK: [[#@LINE+1]]:21: error: unknown token in expression
.bundle_align_mode
imull $17, %ebx, %ebp
-## .bundle_unlock can't come without a .bundle_lock before it
#--- unlock-without-lock.s
+## .bundle_unlock can't come without a .bundle_lock before it
.bundle_align_mode 3
imull $17, %ebx, %ebp
# CHECK: [[#@LINE+1]]:3: error: .bundle_unlock without matching lock
.bundle_unlock
-## .bundle_lock can only take one `align_to_end` flag or no flag.
#--- bad-lock-option.s
+## .bundle_lock can only take one `align_to_end` flag or no flag.
.bundle_align_mode 4
# CHECK: [[#@LINE+1]]:16: error: invalid option for `.bundle_lock`
.bundle_lock 5
imull $17, %ebx, %ebp
.bundle_unlock
+#--- switch-section-locked.s
## This test invokes .bundle_lock and then switches to a different section
## w/o the appropriate unlock.
-#--- switch-section-locked.s
.bundle_align_mode 3
.section text1, "x"
imull $17, %ebx, %ebp
@@ -55,8 +63,8 @@
.section text2, "x"
imull $17, %ebx, %ebp
-## bundle lock size cannot be bigger than the align mode size
#--- group-too-large.s
+## bundle lock size cannot be bigger than the align mode size
.text
foo:
.bundle_align_mode 4
@@ -68,11 +76,11 @@ foo:
callq bar
callq bar
callq bar
-# CHECK: [[#@LINE+1]]:3: error: fragment can't be larger than a bundle size
+# CHECK: [[#@LINE+1]]:3: error: .bundle_lock group is larger than the bundle size
.bundle_unlock
-## test that nested lock is emitting the right error.
#--- nested-lock.s
+## test that nested lock is emitting the right error.
.bundle_align_mode 4
foo:
## repeating .bundle_align_mode with the same value is allowed.
@@ -85,10 +93,12 @@ foo:
.bundle_unlock
.bundle_unlock
-## `.bundle_align_mode 0` is not supported.
-#--- mode-zero.s
-# CHECK: [[#@LINE+1]]:3: error: disabling .bundle_align_mode is not supported
+#--- mode-bad-size.s
+## Unlike GNU as, `.bundle_align_mode 0` (disabling bundling) is not supported.
+# CHECK: [[#@LINE+1]]:22: error: invalid bundle alignment size (expected between 1 and 30)
.bundle_align_mode 0
+# CHECK: [[#@LINE+1]]:22: error: invalid bundle alignment size (expected between 1 and 30)
+ .bundle_align_mode 31
imull $17, %ebx, %ebp
#--- align-in-lock.s
@@ -100,8 +110,42 @@ foo:
# CHECK: [[#@LINE+1]]:3: error: alignment and .org directives are not supported inside a .bundle_lock group
.bundle_unlock
-## Instruction bundling cannot be combined with branch alignment.
+#--- unterminated-lock.s
+## A .bundle_lock left open at end of file is diagnosed rather than ignored.
+ .bundle_align_mode 4
+ int3
+ .bundle_lock
+# CHECK: [[#@LINE+1]]:3: error: unterminated .bundle_lock
+ callq bar
+
+#--- lock-in-data.s
+## Padding a group is only meaningful where nops are instructions.
+ .bundle_align_mode 4
+ .data
+ .byte 1
+# CHECK: [[#@LINE+1]]:3: error: .bundle_lock is only allowed in an executable section
+ .bundle_lock
+ .quad 0
+# CHECK: [[#@LINE+1]]:3: error: .bundle_unlock without matching lock
+ .bundle_unlock
+
+#--- prefix-ends-lock.s
+## A group ending in a prefix cannot keep it attached to its instruction.
+ .bundle_align_mode 4
+ .bundle_lock
+ int3
+ lock
+ .bundle_unlock
+# CHECK: [[#@LINE+1]]:3: error: instruction prefix cannot be the last instruction of a .bundle_lock group
+ cmpxchgl %r12d, (%rbx)
+
+#--- unsupported.s
+# FORMAT: [[#@LINE+2]]:3: error: aligned bundling is not supported by this object file format
+# TARGET: [[#@LINE+1]]:3: error: aligned bundling is not supported by this target
+ .bundle_align_mode 4
+
#--- bundle-with-align-branch.s
+## Instruction bundling cannot be combined with branch alignment.
# CHECK: [[#@LINE+1]]:3: error: .bundle_align_mode is incompatible with branch alignment
.bundle_align_mode 5
imull $17, %ebx, %ebp
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-lock.s b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
index 1e7bbeb4ee73a..0835c02094bfc 100644
--- a/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
@@ -55,3 +55,19 @@ lock_fit_exactly:
.endr
.bundle_unlock
# CHECK: 56: incl
+
+## The instruction after .bundle_unlock needs its own boundary fragment; the
+## prefix before .bundle_lock must not leave a stale one.
+ .align 16, 0x90
+prefix_before_lock:
+ lock
+ .bundle_lock
+ callq bar
+ callq bar
+ .bundle_unlock
+ movl $1, (%rsp)
+# CHECK: 60: lock
+# CHECK-NEXT: 61: callq
+# CHECK-NEXT: 66: callq
+# CHECK-NEXT: 6b: nop
+# CHECK-NEXT: 70: movl
diff --git a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
index 7eafd73ce7654..ee0208b005180 100644
--- a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
+++ b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
@@ -3,6 +3,10 @@
# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/prefix-pad.s
# RUN: llvm-mc -filetype=obj -triple x86_64 %t/no-prefix-pad.s --x86-pad-max-prefix-size=1 \
# RUN: | llvm-objdump -d - | FileCheck %t/no-prefix-pad.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/stay-in-bundle.s --x86-pad-max-prefix-size=5 \
+# RUN: | llvm-objdump -d - | FileCheck %t/stay-in-bundle.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/group-stays-in-bundle.s --x86-pad-max-prefix-size=1 \
+# RUN: | llvm-nm --numeric-sort - | FileCheck %t/group-stays-in-bundle.s
#--- prefix-pad.s
.section text1, "x"
@@ -95,8 +99,8 @@ ensure_bundle_boundary:
# CHECK-NEXT: 34: call
# CHECK-NEXT: 3b: call
-## This test contains instructions that must not be padded.
#--- no-prefix-pad.s
+## This test contains instructions that must not be padded.
.text
.bundle_align_mode 5
prefix_cmpxchg_in_a_bundle:
@@ -124,3 +128,38 @@ no_pad_before_after_prefix:
cmpxchgl %r12d, __thread_list_lock(%rip)
# CHECK: 40: f0 lock
# CHECK-NEXT: 41: 44 0f b1 25 00 00 00 00 cmpxchgl %r12d, (%rip)
+
+#--- stay-in-bundle.s
+## `.nops` spans a bundle boundary without starting on one. Padding must not
+## reach the instructions in the earlier bundle.
+ .bundle_align_mode 3
+ .rept 7
+ int3
+ .endr
+ .nops 30
+ .bundle_lock
+ callq bar
+ .bundle_unlock
+## The int3 at 0x6 must keep its 1-byte encoding: growing it would run past 0x8.
+# CHECK: 6: cc {{ *}}int3
+# CHECK-NEXT: 7: 90 {{ *}}nop
+# CHECK: 28: e8
+
+#--- group-stays-in-bundle.s
+## Absorbing padding moves the group's start earlier; it must not grow past its
+## bundle.
+ .bundle_align_mode 3
+ .bundle_lock
+gs:
+ addl %ebp, %eax
+ int3
+ movq %rax, %rbx
+ int3
+ge:
+ .bundle_unlock
+ .space 18, 0x90
+ .bundle_lock align_to_end
+ .bundle_unlock
+## The 7-byte group has to stay within the bundle at [0, 8).
+# CHECK: 0000000000000000 {{.*}} gs
+# CHECK-NEXT: 0000000000000007 {{.*}} ge
diff --git a/llvm/test/MC/X86/AlignedBundling/section.s b/llvm/test/MC/X86/AlignedBundling/section.s
index 5be4acf5cb9d2..c04907cbab3b9 100644
--- a/llvm/test/MC/X86/AlignedBundling/section.s
+++ b/llvm/test/MC/X86/AlignedBundling/section.s
@@ -8,8 +8,8 @@
# RUN: llvm-mc -filetype=obj -triple x86_64 %t/data-section-alignment.s \
# RUN: | llvm-readobj --sections - | FileCheck %t/data-section-alignment.s
-## Test two different executable sections with bundling.
#--- two-sections.s
+## Test two different executable sections with bundling.
.bundle_align_mode 3
.section text1, "x"
# CHECK: section text1
@@ -29,8 +29,8 @@
# CHECK: 6: nop
# CHECK-NEXT: 8: imull
-## Test that bundle-aligned sections with instructions are aligned
#--- section-alignment.s
+## Test that bundle-aligned sections with instructions are aligned
.bundle_align_mode 5
# CHECK: Sections
## Check that the empty .text section has the default alignment
@@ -50,8 +50,8 @@
# CHECK-NOT: Name
# CHECK: AddressAlignment: 32
-## Test that bundle alignment is only applied to executable sections.
#--- data-section-alignment.s
+## Test that bundle alignment is only applied to executable sections.
.bundle_align_mode 5
.text
imull $17, %ebx, %ebp
>From 92ae0fbd5914c3ae66d4dfdabbfa2761d4993c87 Mon Sep 17 00:00:00 2001
From: Zachary Yedidia <zyedidia at gmail.com>
Date: Sun, 2 Aug 2026 15:15:59 -0700
Subject: [PATCH 24/24] Resolve conflict with reset
---
llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 7 -------
1 file changed, 7 deletions(-)
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 2105b9e4ea858..5d245f7f90cf3 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -176,13 +176,6 @@ class X86AsmBackend : public MCAsmBackend {
PrevInstPosition = {};
}
- void reset() override {
- PrevInst = MCInst();
- PrevInstOpcode = 0;
- PendingBA = nullptr;
- PrevInstPosition = {};
- }
-
void emitInstructionBegin(MCObjectStreamer &OS, const MCInst &Inst,
const MCSubtargetInfo &STI);
void emitInstructionEnd(MCObjectStreamer &OS, const MCInst &Inst);
More information about the llvm-commits
mailing list