[llvm] [X86] Reimplement bundle alignment mode (PR #175830)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 10:02:54 PDT 2026
https://github.com/t-noh updated https://github.com/llvm/llvm-project/pull/175830
>From 2b9f15225ab0dc71f16bce8b2cf1eb6bc184baa8 Mon Sep 17 00:00:00 2001
From: Taehyun Noh <taehyun at utexas.edu>
Date: Mon, 12 Jan 2026 16:22:12 -0800
Subject: [PATCH] [X86] Reimplement bundle alignment mode
Reintroduce bundle alignment for X86 using a cleaner architecture that
builds on MCBoundaryAlignFragment infrastructure rather than per-fragment
padding fields. This preserves the MC simplifications from the original
removal while restoring functionality.
Key differences from the original implementation:
- Uses MCBoundaryAlignFragment instead of per-fragment BundlePadding
- Does not override emitInstToData (remains non-virtual)
- Adds prefix padding optimization (reuses --x86-pad-max-prefix-size)
- NOPs never span bundle boundaries
- X86-only
Co-Authored-By: Zachary Yedidia <zyedidia at gmail.com>
Co-Authored-By: Taehyun Noh <taehyun at utexas.edu>
---
llvm/include/llvm/MC/MCAssembler.h | 9 +
llvm/include/llvm/MC/MCELFStreamer.h | 8 +
llvm/include/llvm/MC/MCSection.h | 14 ++
llvm/include/llvm/MC/MCStreamer.h | 13 +
llvm/lib/MC/MCAsmStreamer.cpp | 21 ++
llvm/lib/MC/MCAssembler.cpp | 96 ++++++--
llvm/lib/MC/MCELFStreamer.cpp | 92 ++++++++
llvm/lib/MC/MCFragment.cpp | 4 +-
llvm/lib/MC/MCObjectStreamer.cpp | 16 ++
llvm/lib/MC/MCParser/AsmParser.cpp | 68 ++++++
llvm/lib/MC/MCStreamer.cpp | 3 +
.../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 222 ++++++++++++++++--
.../asm-printing-bundle-directives.s | 20 ++
.../X86/AlignedBundling/bundle-after-relax.s | 53 +++++
.../bundle-align-to-end-lock.s | 65 +++++
.../MC/X86/AlignedBundling/bundle-errors.s | 78 ++++++
.../test/MC/X86/AlignedBundling/bundle-inst.s | 36 +++
.../test/MC/X86/AlignedBundling/bundle-lock.s | 57 +++++
.../MC/X86/AlignedBundling/long-nop-pad.s | 40 ++++
.../MC/X86/AlignedBundling/prefix-padding.s | 69 ++++++
.../relax-for-prefix-padding.s | 48 ++++
llvm/test/MC/X86/AlignedBundling/section.s | 49 ++++
22 files changed, 1037 insertions(+), 44 deletions(-)
create mode 100644 llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-errors.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-inst.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/bundle-lock.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/prefix-padding.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
create mode 100644 llvm/test/MC/X86/AlignedBundling/section.s
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 22f8ebde88756..2ede309321631 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -64,6 +64,8 @@ class MCAssembler {
// forward-reference displacements in `evaluateFixup`.
int64_t Stretch = 0;
+ unsigned BundleAlignSize = 0;
+
SectionListType Sections;
SmallVector<const MCSymbol *, 0> Symbols;
@@ -122,6 +124,9 @@ class MCAssembler {
void relaxDwarfCallFrameFragment(MCFragment &F);
void relaxSFrameFragment(MCFragment &DF);
+ /// Compute the padding size to boundary-align its connected fragments.
+ uint64_t computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF);
+
public:
/// Construct a new assembler instance.
//
@@ -198,6 +203,10 @@ class MCAssembler {
void setRelaxAll(bool Value) { RelaxAll = Value; }
int64_t getStretch() const { return Stretch; }
+ bool isBundlingEnabled() const { return BundleAlignSize != 0; }
+ unsigned getBundleAlignSize() const { return BundleAlignSize; }
+ void setBundleAlignSize(unsigned Size) { BundleAlignSize = Size; }
+
const_iterator begin() const { return Sections.begin(); }
const_iterator end() const { return Sections.end(); }
diff --git a/llvm/include/llvm/MC/MCELFStreamer.h b/llvm/include/llvm/MC/MCELFStreamer.h
index 5c3246e42c3f3..e0486362c5435 100644
--- a/llvm/include/llvm/MC/MCELFStreamer.h
+++ b/llvm/include/llvm/MC/MCELFStreamer.h
@@ -69,6 +69,13 @@ class MCELFStreamer : public MCObjectStreamer {
void emitCGProfileEntry(const MCSymbolRefExpr *From,
const MCSymbolRefExpr *To, uint64_t Count) override;
+ void emitBundleAlignMode(Align Alignment) override;
+ void emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) override;
+ void emitBundleUnlock(const MCSubtargetInfo &STI) override;
+ bool isBundleLocked() const {
+ return getCurrentSectionOnly()->isBundleLocked();
+ }
+
// This is final. Override MCTargetStreamer::finish instead for
// target-specific code.
void finishImpl() final;
@@ -132,6 +139,7 @@ class MCELFStreamer : public MCObjectStreamer {
// GNU attributes that will get emitted at the end of the asm file.
SmallVector<AttributeItem, 64> GNUAttributes;
+ MCBoundaryAlignFragment *BundleBA = nullptr;
public:
void emitGNUAttribute(unsigned Tag, unsigned Value) override {
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 82bfa41c9215b..27f16eab0c6b9 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -541,6 +541,10 @@ class MCBoundaryAlignFragment : public MCFragment {
/// is not meaningful before that.
uint64_t Size = 0;
+ /// If true, align the last instruction in the fragment to the end of the
+ /// fragment.
+ bool AlignToEnd = false;
+
public:
MCBoundaryAlignFragment(Align AlignBoundary, const MCSubtargetInfo &STI)
: MCFragment(FT_BoundaryAlign), AlignBoundary(AlignBoundary) {
@@ -553,6 +557,9 @@ class MCBoundaryAlignFragment : public MCFragment {
Align getAlignment() const { return AlignBoundary; }
void setAlignment(Align Value) { AlignBoundary = Value; }
+ bool isAlignToEnd() const { return AlignToEnd; }
+ void setAlignToEnd(bool Value) { AlignToEnd = Value; }
+
const MCFragment *getLastFragment() const { return LastFragment; }
void setLastFragment(const MCFragment *F) {
assert(!F || getParent() == F->getParent());
@@ -606,6 +613,10 @@ class LLVM_ABI MCSection {
// fragment may not be fully resolved.
unsigned FirstLinkerRelaxable = -1u;
+ /// If bundle-locked, we ensure all instructions in the section are placed in
+ /// the same bundle.
+ bool IsBundleLocked = false;
+
/// Whether this section has had instructions emitted into it.
bool HasInstructions : 1;
@@ -672,6 +683,9 @@ class LLVM_ABI MCSection {
bool isLinkerRelaxable() const { return FirstLinkerRelaxable != -1u; }
void setFirstLinkerRelaxable(unsigned Order) { FirstLinkerRelaxable = Order; }
+ bool isBundleLocked() const { return IsBundleLocked; }
+ void setIsBundleLocked(bool Value) { IsBundleLocked = Value; }
+
MCFragment &getDummyFragment() { return DummyFragment; }
FragList *curFragList() const { return CurFragList; }
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 535e9788d56dd..34875d34685eb 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -1094,6 +1094,19 @@ class LLVM_ABI MCStreamer {
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym);
+ /// Set the bundle alignment mode from now on in the section.
+ /// The value 1 means turn the bundle alignment off.
+ virtual void emitBundleAlignMode(Align Alignment);
+
+ /// The following instructions are a bundle-locked group.
+ ///
+ /// \param AlignToEnd - If true, the bundle-locked group will be aligned to
+ /// the end of a bundle.
+ virtual void emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI);
+
+ /// Ends a bundle-locked group.
+ virtual void emitBundleUnlock(const MCSubtargetInfo &STI);
+
/// If this file is backed by a assembly streamer, this dumps the
/// specified string in the output .s file. This capability is indicated by
/// the hasRawTextSupport() predicate. By default this aborts.
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 5c72a883c062e..fe484a4e6a7e6 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -72,6 +72,9 @@ class MCAsmStreamer final : public MCAsmBaseStreamer {
raw_svector_ostream &OS) const;
void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
+ void emitBundleAlignMode(Align Alignment) override;
+ void emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) override;
+ void emitBundleUnlock(const MCSubtargetInfo &STI) override;
/// Helper to emit common .loc directive flags, isa, and discriminator.
void emitDwarfLocDirectiveFlags(unsigned Flags, unsigned Isa,
@@ -2570,6 +2573,24 @@ void MCAsmStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index,
EmitEOL();
}
+void MCAsmStreamer::emitBundleAlignMode(Align Alignment) {
+ OS << "\t.bundle_align_mode " << Log2(Alignment);
+ EmitEOL();
+}
+
+void MCAsmStreamer::emitBundleLock(bool AlignToEnd,
+ const MCSubtargetInfo &STI) {
+ OS << "\t.bundle_lock";
+ if (AlignToEnd)
+ OS << " align_to_end";
+ EmitEOL();
+}
+
+void MCAsmStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
+ OS << "\t.bundle_unlock";
+ EmitEOL();
+}
+
void MCAsmStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
const MCExpr *Expr, SMLoc) {
OS << "\t.reloc ";
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 62f248dd2e481..f99492f872d6c 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -396,6 +396,35 @@ void MCAssembler::addRelocDirective(RelocDirective RD) {
relocDirectives.push_back(RD);
}
+/// Write NOPs while limiting the maximum NOP size.
+static void writeControlledNops(raw_ostream &OS, const MCAssembler &Asm,
+ uint64_t NumBytes, uint64_t FragmentOffset,
+ uint64_t MaxNopSize,
+ const MCSubtargetInfo *STI) {
+ uint64_t NumBytesEmitted = 0;
+ while (NumBytesEmitted < NumBytes) {
+ uint64_t NumBytesToEmit = std::min(NumBytes - NumBytesEmitted, MaxNopSize);
+
+ if (Asm.isBundlingEnabled()) {
+ unsigned BundleAlignSize = Asm.getBundleAlignSize();
+ uint64_t OffsetInBundle =
+ (FragmentOffset + NumBytesEmitted) & (BundleAlignSize - 1);
+ uint64_t SpaceInBundle = BundleAlignSize - OffsetInBundle;
+ NumBytesToEmit = std::min(NumBytesToEmit, SpaceInBundle);
+ }
+
+ assert(NumBytesToEmit && "try to emit zero-sized NOP");
+
+ if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit, STI)) {
+ report_fatal_error("unable to write NOP sequence of the remaining " +
+ Twine(NumBytesToEmit) + " bytes");
+ return;
+ }
+
+ NumBytesEmitted += NumBytesToEmit;
+ }
+}
+
/// Write the fragment \p F to the output file.
static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
const MCFragment &F) {
@@ -544,26 +573,22 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
if (!ControlledNopLength)
ControlledNopLength = MaximumNopLength;
- while (NumBytes) {
- uint64_t NumBytesToEmit =
- (uint64_t)std::min(NumBytes, ControlledNopLength);
- assert(NumBytesToEmit && "try to emit empty NOP instruction");
- if (!Asm.getBackend().writeNopData(OS, NumBytesToEmit,
- NF.getSubtargetInfo())) {
- report_fatal_error("unable to write nop sequence of the remaining " +
- Twine(NumBytesToEmit) + " bytes");
- break;
- }
- NumBytes -= NumBytesToEmit;
- }
+ writeControlledNops(OS, Asm, (uint64_t)NumBytes, Asm.getFragmentOffset(NF),
+ (uint64_t)ControlledNopLength, NF.getSubtargetInfo());
break;
}
case MCFragment::FT_BoundaryAlign: {
const MCBoundaryAlignFragment &BF = cast<MCBoundaryAlignFragment>(F);
- if (!Asm.getBackend().writeNopData(OS, FragmentSize, BF.getSubtargetInfo()))
- report_fatal_error("unable to write nop sequence of " +
- Twine(FragmentSize) + " bytes");
+ if (!Asm.isBundlingEnabled()) {
+ if (!Asm.getBackend().writeNopData(OS, FragmentSize,
+ BF.getSubtargetInfo()))
+ report_fatal_error("unable to write nop sequence of " +
+ Twine(FragmentSize) + " bytes");
+ } else {
+ writeControlledNops(OS, Asm, FragmentSize, Asm.getFragmentOffset(BF),
+ FragmentSize, BF.getSubtargetInfo());
+ }
break;
}
@@ -950,11 +975,10 @@ static bool needPadding(uint64_t StartAddr, uint64_t Size,
isAgainstBoundary(StartAddr, Size, BoundaryAlignment);
}
-void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
- // BoundaryAlignFragment that doesn't need to align any fragment should not be
- // relaxed.
+uint64_t
+MCAssembler::computeBoundaryAlignSize(const MCBoundaryAlignFragment &BF) {
if (!BF.getLastFragment())
- return;
+ return 0;
uint64_t AlignedOffset = getFragmentOffset(BF);
uint64_t AlignedSize = 0;
@@ -965,9 +989,32 @@ void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
}
Align BoundaryAlignment = BF.getAlignment();
- uint64_t NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
- ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
- : 0U;
+
+ uint64_t NewSize = 0;
+ if (isBundlingEnabled()) {
+ // For bundle alignment, we only pad instructions that cross the boundary.
+ NewSize = mayCrossBoundary(AlignedOffset, AlignedSize, BoundaryAlignment)
+ ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
+ : 0U;
+ if (BF.isAlignToEnd()) {
+ NewSize =
+ offsetToAlignment(AlignedOffset + AlignedSize, BoundaryAlignment);
+ }
+ } else {
+ NewSize = needPadding(AlignedOffset, AlignedSize, BoundaryAlignment)
+ ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
+ : 0U;
+ }
+ return NewSize;
+}
+
+void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
+ // BoundaryAlignFragment that doesn't need to align any fragment should not be
+ // relaxed.
+ if (!BF.getLastFragment())
+ return;
+
+ uint64_t NewSize = computeBoundaryAlignSize(BF);
if (NewSize == BF.getSize())
return;
BF.setSize(NewSize);
@@ -1035,7 +1082,10 @@ void MCAssembler::relaxFragment(MCFragment &F) {
relaxAlign(F);
break;
case MCFragment::FT_Relaxable:
- assert(!getRelaxAll() && "Did not expect a FT_Relaxable in RelaxAll mode");
+ // Bundling emits every instruction as relaxable, so FT_Relaxable is
+ // expected with RelaxAll mode once bundling is enabled.
+ assert((isBundlingEnabled() || !getRelaxAll()) &&
+ "Did not expect a FT_Relaxable in RelaxAll mode");
relaxInstruction(F);
break;
case MCFragment::FT_LEB:
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index d584be81440b2..5b80b547cb05b 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -80,6 +80,22 @@ void MCELFStreamer::emitLabelAtPos(MCSymbol *S, SMLoc Loc, MCFragment &F,
void MCELFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
MCAssembler &Asm = getAssembler();
+ MCFragment *CF = getCurrentFragment();
+ if (Asm.isBundlingEnabled()) {
+ if (isBundleLocked()) {
+ getContext().reportError(
+ getStartTokLoc(),
+ "unterminated .bundle_lock when changing a section");
+ // Clean up bundle state to allow continuing.
+ MCSection *CurSec = CF->getParent();
+ CurSec->setIsBundleLocked(false);
+ BundleBA = nullptr;
+ }
+
+ // Ensure the previous section gets aligned if necessary.
+ if (Asm.isBundlingEnabled() && CF->getParent()->hasInstructions())
+ Section->ensureMinAlignment(Align(Asm.getBundleAlignSize()));
+ }
auto *SectionELF = static_cast<const MCSectionELF *>(Section);
const MCSymbol *Grp = SectionELF->getGroup();
if (Grp)
@@ -316,6 +332,82 @@ void MCELFStreamer::emitIdent(StringRef IdentString) {
popSection();
}
+void MCELFStreamer::emitBundleAlignMode(Align Alignment) {
+ if (Log2(Alignment) > 30)
+ getContext().reportError(getStartTokLoc(),
+ ".bundle_align_mode alignment must be <= 30");
+ MCAssembler &Assembler = getAssembler();
+ setAllowAutoPadding(true);
+
+ if (Alignment > 1 && (Assembler.getBundleAlignSize() == 0 ||
+ Assembler.getBundleAlignSize() == Alignment.value()))
+ Assembler.setBundleAlignSize(Alignment.value());
+ else
+ getContext().reportError(getStartTokLoc(),
+ ".bundle_align_mode cannot be changed once set");
+}
+
+void MCELFStreamer::emitBundleLock(bool AlignToEnd,
+ const MCSubtargetInfo &STI) {
+ MCSection &Sec = *getCurrentSectionOnly();
+ auto &Asm = getAssembler();
+
+ if (!Asm.isBundlingEnabled()) {
+ getContext().reportError(
+ getStartTokLoc(), ".bundle_lock forbidden when bundling is disabled");
+ return;
+ }
+
+ if (Sec.isBundleLocked()) {
+ getContext().reportError(getStartTokLoc(),
+ "nested .bundle_lock is not allowed");
+ return;
+ }
+ Sec.setIsBundleLocked(true);
+
+ auto AlignBoundary = Asm.getBundleAlignSize();
+ BundleBA =
+ newSpecialFragment<MCBoundaryAlignFragment>(Align(AlignBoundary), STI);
+ BundleBA->setAlignToEnd(AlignToEnd);
+}
+
+void MCELFStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {
+ MCSection &Sec = *getCurrentSectionOnly();
+
+ if (!getAssembler().isBundlingEnabled()) {
+ getContext().reportError(
+ getStartTokLoc(), ".bundle_unlock forbidden when bundling is disabled");
+ return;
+ }
+ if (!isBundleLocked()) {
+ getContext().reportError(getStartTokLoc(),
+ ".bundle_unlock without matching lock");
+ return;
+ }
+
+ Sec.setIsBundleLocked(false);
+
+ MCFragment *CF = getCurrentFragment();
+ BundleBA->setLastFragment(CF);
+ // Bundle overflow check.
+ uint64_t AlignedSize = 0;
+ for (const MCFragment *F = BundleBA->getNext();; F = F->getNext()) {
+ AlignedSize += getAssembler().computeFragmentSize(*F);
+ if (F == BundleBA->getLastFragment())
+ break;
+ }
+ BundleBA = nullptr;
+
+ if (AlignedSize > getAssembler().getBundleAlignSize())
+ getContext().reportError(getStartTokLoc(),
+ "fragment can't be larger than a bundle size");
+
+ newFragment();
+
+ CF->getParent()->ensureMinAlignment(
+ Align(getAssembler().getBundleAlignSize()));
+}
+
void MCELFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *Sym,
uint64_t Offset,
const MCSymbolRefExpr *&SRE) {
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index 21a304da0bb4f..78c955e4f3d1a 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -167,8 +167,8 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
case MCFragment::FT_BoundaryAlign: {
const auto *BF = cast<MCBoundaryAlignFragment>(this);
OS << " BoundarySize:" << BF->getAlignment().value()
- << " LastFragment:" << BF->getLastFragment()
- << " Size:" << BF->getSize();
+ << " LastFragment:" << BF->getLastFragment() << " Size:" << BF->getSize()
+ << " AlignToEnd:" << BF->isAlignToEnd();
break;
}
case MCFragment::FT_PrefAlign:
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 6cbf208fad20d..6603de22750b3 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -412,6 +412,22 @@ void MCObjectStreamer::emitInstruction(const MCInst &Inst,
// If this instruction doesn't need relaxation, just emit it as data.
MCAssembler &Assembler = getAssembler();
MCAsmBackend &Backend = Assembler.getBackend();
+
+ // To enable better bundle-nop optimization, we emit every instruction
+ // as a relaxable fragment.
+ if (Assembler.isBundlingEnabled()) {
+ if (Sec->isBundleLocked() || Assembler.getRelaxAll()) {
+ MCInst Relaxed = Inst;
+ while (Backend.mayNeedRelaxation(Relaxed.getOpcode(),
+ Relaxed.getOperands(), STI))
+ Backend.relaxInstruction(Relaxed, STI);
+ emitInstToFragment(Relaxed, STI);
+ } else {
+ emitInstToFragment(Inst, STI);
+ }
+ return;
+ }
+
if (!(Backend.mayNeedRelaxation(Inst.getOpcode(), Inst.getOperands(), STI) ||
Backend.allowEnhancedRelaxation())) {
emitInstToData(Inst, STI);
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 4e95bf47bb7ee..8fde267bfea0d 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -422,6 +422,9 @@ class AsmParser : public MCAsmParser {
DK_ORG,
DK_FILL,
DK_ENDR,
+ DK_BUNDLE_ALIGN_MODE,
+ DK_BUNDLE_LOCK,
+ DK_BUNDLE_UNLOCK,
DK_ZERO,
DK_EXTERN,
DK_GLOBL,
@@ -703,6 +706,13 @@ class AsmParser : public MCAsmParser {
bool parseDirectiveAddrsig();
bool parseDirectiveAddrsigSym();
+ // ".bundle_align_mode"
+ bool parseDirectiveBundleAlignMode();
+ // ".bundle_lock"
+ bool parseDirectiveBundleLock();
+ // ".bundle_unlock"
+ bool parseDirectiveBundleUnlock();
+
void initializeDirectiveKindMap();
void initializeCVDefRangeTypeMap();
};
@@ -2065,6 +2075,12 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
return parseDirectiveIrpc(IDLoc);
case DK_ENDR:
return parseDirectiveEndr(IDLoc);
+ case DK_BUNDLE_ALIGN_MODE:
+ return parseDirectiveBundleAlignMode();
+ case DK_BUNDLE_LOCK:
+ return parseDirectiveBundleLock();
+ case DK_BUNDLE_UNLOCK:
+ return parseDirectiveBundleUnlock();
case DK_SLEB128:
return parseDirectiveLEB128(true);
case DK_ULEB128:
@@ -5521,6 +5537,9 @@ void AsmParser::initializeDirectiveKindMap() {
DirectiveKindMap[".irp"] = DK_IRP;
DirectiveKindMap[".irpc"] = DK_IRPC;
DirectiveKindMap[".endr"] = DK_ENDR;
+ DirectiveKindMap[".bundle_align_mode"] = DK_BUNDLE_ALIGN_MODE;
+ DirectiveKindMap[".bundle_lock"] = DK_BUNDLE_LOCK;
+ DirectiveKindMap[".bundle_unlock"] = DK_BUNDLE_UNLOCK;
DirectiveKindMap[".if"] = DK_IF;
DirectiveKindMap[".ifeq"] = DK_IFEQ;
DirectiveKindMap[".ifge"] = DK_IFGE;
@@ -5875,6 +5894,55 @@ bool AsmParser::parseDirectiveAddrsigSym() {
return false;
}
+/// parseDirectiveBundleAlignMode
+/// ::= {.bundle_align_mode} expression
+bool AsmParser::parseDirectiveBundleAlignMode() {
+ // Expect a single argument: an expression that evaluates to a constant
+ // in the inclusive range 0-30.
+ SMLoc ExprLoc = getLexer().getLoc();
+ int64_t AlignSizePow2;
+ if (checkForValidSection() || parseAbsoluteExpression(AlignSizePow2) ||
+ parseEOL() ||
+ check(AlignSizePow2 < 0 || AlignSizePow2 > 30, ExprLoc,
+ "invalid bundle alignment size (expected between 0 and 30)"))
+ return true;
+
+ getStreamer().emitBundleAlignMode(Align(1ULL << AlignSizePow2));
+ return false;
+}
+
+/// parseDirectiveBundleLock
+/// ::= {.bundle_lock} [align_to_end]
+bool AsmParser::parseDirectiveBundleLock() {
+ if (checkForValidSection())
+ return true;
+ bool AlignToEnd = false;
+
+ StringRef Option;
+ SMLoc Loc = getTok().getLoc();
+ const char *InvalidOptionError = "invalid option for `.bundle_lock`";
+
+ if (!parseOptionalToken(AsmToken::EndOfStatement)) {
+ if (check(parseIdentifier(Option), Loc, InvalidOptionError) ||
+ check(Option != "align_to_end", Loc, InvalidOptionError) || parseEOL())
+ return true;
+ AlignToEnd = true;
+ }
+
+ getStreamer().emitBundleLock(AlignToEnd, getTargetParser().getSTI());
+ return false;
+}
+
+/// parseDirectiveBundleUnlock
+/// ::= {.bundle_unlock}
+bool AsmParser::parseDirectiveBundleUnlock() {
+ if (checkForValidSection() || parseEOL())
+ return true;
+
+ getStreamer().emitBundleUnlock(getTargetParser().getSTI());
+ return false;
+}
+
bool AsmParser::parseDirectivePseudoProbe() {
int64_t Guid;
int64_t Index;
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index b52e3e5b90bf2..799bea075fe7d 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -1369,6 +1369,9 @@ void MCStreamer::emitCodeAlignment(Align Alignment, const MCSubtargetInfo *STI,
unsigned MaxBytesToEmit) {}
void MCStreamer::emitValueToOffset(const MCExpr *Offset, unsigned char Value,
SMLoc Loc) {}
+void MCStreamer::emitBundleAlignMode(Align Alignment) {}
+void MCStreamer::emitBundleLock(bool AlignToEnd, const MCSubtargetInfo &STI) {}
+void MCStreamer::emitBundleUnlock(const MCSubtargetInfo &STI) {}
void MCStreamer::finishImpl() {}
bool MCStreamer::popSection() {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 7515c75ab33e1..2b7d38774c245 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -133,6 +133,8 @@ class X86AsmBackend : public MCAsmBackend {
bool needAlign(const MCInst &Inst) const;
bool canPadBranches(MCObjectStreamer &OS) const;
bool canPadInst(const MCInst &Inst, MCObjectStreamer &OS) const;
+ void emitInstructionBeginBundle(MCObjectStreamer &OS);
+ void emitInstructionEndBundle(MCObjectStreamer &OS);
public:
X86AsmBackend(const Target &T, const MCSubtargetInfo &STI)
@@ -197,6 +199,12 @@ class X86AsmBackend : public MCAsmBackend {
bool finishLayout() const override;
+ bool padInstsBackward(SmallVectorImpl<MCFragment *> &Relaxable,
+ unsigned &RemainingSize) const;
+ bool dividePadInBundle(const MCAssembler &Asm,
+ ArrayRef<MCFragment *> Peephole) const;
+ bool optimizeBundleNops(const MCAssembler &Asm) const;
+
unsigned getMaximumNopSize(const MCSubtargetInfo &STI) const override;
bool writeNopData(raw_ostream &OS, uint64_t Count,
@@ -463,9 +471,55 @@ void X86_MC::emitInstruction(MCObjectStreamer &S, const MCInst &Inst,
Backend.emitInstructionEnd(S, Inst);
}
+/// If the upcoming instruction is inside the bundle lock, do nothing so that
+/// the ObjectStreamer emits the instruction to the current fragment. If not, it
+/// creates a new BA to group bundled fragments.
+void X86AsmBackend::emitInstructionBeginBundle(MCObjectStreamer &OS) {
+ assert(Asm->isBundlingEnabled());
+
+ if (OS.getCurrentSectionOnly()->isBundleLocked()) {
+ OS.getCurrentFragment()->setAllowAutoPadding(true);
+ return;
+ }
+ PendingBA = OS.newSpecialFragment<MCBoundaryAlignFragment>(
+ Align(Asm->getBundleAlignSize()), STI);
+ // We can set LastFragment now, before the instruction is emitted, as bundling
+ // emits one fragment per instruction. Deferring setLastFragment to
+ // post-emitInstruction would risk capturing a fragment that a subsequent
+ // emitCodeAlignment repurposes in-place to FT_Align, corrupting the BA's
+ // boundary range.
+ PendingBA->setLastFragment(OS.getCurrentFragment());
+
+ OS.getCurrentFragment()->setAllowAutoPadding(true);
+}
+
+/// If the just-emitted instruction is inside the bundle lock, check the current
+/// fragment is non-zero to ensure the instruction is placed as expected. If it
+/// is not locked, finalize pending BA Fragment. emitBundleUnlock will close the
+/// fragment and start a new empty fragment.
+void X86AsmBackend::emitInstructionEndBundle(MCObjectStreamer &OS) {
+ assert(Asm->isBundlingEnabled());
+
+ MCFragment *CF = OS.getCurrentFragment();
+
+ if (OS.getCurrentSectionOnly()->isBundleLocked()) {
+ // We're still inside the lock, do not close the current fragment with BA.
+ return;
+ }
+ assert(PendingBA && "MCBoundaryAlignFragment is expected for every "
+ "instruction if it is not bundle-locked");
+
+ PendingBA = nullptr;
+
+ CF->getParent()->ensureMinAlignment(Align(Asm->getBundleAlignSize()));
+}
+
/// Insert BoundaryAlignFragment before instructions to align branches.
void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
- const MCInst &Inst, const MCSubtargetInfo &STI) {
+ const MCInst &Inst,
+ const MCSubtargetInfo &STI) {
+ if (Asm->isBundlingEnabled())
+ return emitInstructionBeginBundle(OS);
bool CanPadInst = canPadInst(Inst, OS);
if (CanPadInst)
OS.getCurrentFragment()->setAllowAutoPadding(true);
@@ -528,6 +582,8 @@ void X86AsmBackend::emitInstructionBegin(MCObjectStreamer &OS,
/// Set the last fragment to be aligned for the BoundaryAlignFragment.
void X86AsmBackend::emitInstructionEnd(MCObjectStreamer &OS,
const MCInst &Inst) {
+ if (Asm->isBundlingEnabled())
+ return emitInstructionEndBundle(OS);
// Update PrevInstOpcode here, canPadInst() reads that.
MCFragment *CF = OS.getCurrentFragment();
PrevInstOpcode = Inst.getOpcode();
@@ -735,6 +791,14 @@ bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFragment &,
const MCValue &Target,
uint64_t Value,
bool Resolved) const {
+ if (Asm->isBundlingEnabled() && Resolved) {
+ // This ensures remaining short branches have sufficient headroom to survive
+ // any intra-bundle shift caused by prefix padding in dividePadInBundle.
+ auto BundleAlignSize = Asm->getBundleAlignSize();
+ return (!isInt<8>(Value + BundleAlignSize) ||
+ !isInt<8>(Value - BundleAlignSize)) ||
+ Target.getSpecifier();
+ }
// If resolved, relax if the value is too big for a (signed) i8.
//
// Currently, `jmp local at plt` relaxes JMP even if the offset is small,
@@ -852,7 +916,144 @@ bool X86AsmBackend::padInstructionEncoding(MCFragment &RF,
return Changed;
}
+bool X86AsmBackend::padInstsBackward(SmallVectorImpl<MCFragment *> &Relaxable,
+ unsigned &RemainingSize) const {
+ bool Changed = false;
+ while (!Relaxable.empty() && RemainingSize != 0) {
+ auto &RF = *Relaxable.pop_back_val();
+ // Give the backend a chance to play any tricks it wishes to increase
+ // the encoding size of the given instruction. Target independent code
+ // will try further relaxation, but target's may play further tricks.
+ Changed |= padInstructionEncoding(RF, Asm->getEmitter(), RemainingSize);
+
+ // If we have an instruction which hasn't been fully relaxed, we can't
+ // skip past it and insert bytes before it. Changing its starting
+ // offset might require a larger negative offset than it can encode.
+ // We don't need to worry about larger positive offsets as none of the
+ // possible offsets between this and our align are visible, and the
+ // ones afterwards aren't changing.
+ if (mayNeedRelaxation(RF.getOpcode(), RF.getOperands(),
+ *RF.getSubtargetInfo()))
+ break;
+ }
+ Relaxable.clear();
+ return Changed;
+}
+
+// Peephole is a list of Fragments that ends with non-zero-sized
+// BoundaryAlignFragment. Most of the time it will be every instruction within a
+// bundle, but there can be a partial bundle if it has nops in the middle(e.g.,
+// align_to_end).
+bool X86AsmBackend::dividePadInBundle(const MCAssembler &Asm,
+ ArrayRef<MCFragment *> Peephole) const {
+ bool Changed = false;
+
+ // Last Fragment is either FT_Align or FT_BoundaryAlign
+ auto *LastF = Peephole.back();
+ unsigned RemainingSize =
+ Asm.computeFragmentSize(*LastF) - LastF->getFixedSize();
+
+ unsigned StartOffset = Asm.getFragmentOffset(*LastF);
+ unsigned EndOffset = StartOffset + RemainingSize;
+ auto BoundaryAlignment = Align(Asm.getBundleAlignSize());
+ bool CrossBoundary = (StartOffset >> Log2(BoundaryAlignment)) !=
+ ((EndOffset - 1) >> Log2(BoundaryAlignment));
+
+ if (CrossBoundary) {
+ // i.e., this pad is a mix of suffix fragment of one bundle + prefix of the
+ // very next bundle. It prevents overflow of the first bundle when Peephole
+ // contains more than one bundle.
+ //
+ // This design limits the possibly further-optimized code, which might be
+ // achieved by migrating some instructions to the next bundle, but doing
+ // such may cause fixup errors because instructions can shift by more than
+ // a bundle-size and labels may become unreachable. Until we come up with a
+ // better logic, we limits the optimization scope to a single bundle.
+ RemainingSize -= EndOffset % Asm.getBundleAlignSize();
+ }
+ assert(RemainingSize > 0);
+
+ SmallVector<MCFragment *, 4> Relaxable;
+ for (auto *FIB : Peephole) {
+ if (FIB->getKind() == MCFragment::FT_Data) // Skip and ignore
+ continue;
+
+ if (FIB->getKind() == MCFragment::FT_Align) {
+ // p2align within a bundle
+ Relaxable.clear();
+ continue;
+ }
+
+ if (FIB->getKind() == MCFragment::FT_Relaxable) {
+ auto &RF = cast<MCFragment>(*FIB);
+ Relaxable.push_back(&RF);
+ continue;
+ }
+ }
+
+ // First, try padding previous instructions.
+ Changed |= padInstsBackward(Relaxable, RemainingSize);
+
+ // Second, try padding following instructions.
+ auto padInstsForward = [&](unsigned &Size) {
+ auto *BF = cast<MCBoundaryAlignFragment>(LastF);
+ for (auto *F = BF->getNext();; F = F->getNext()) {
+ if (F->getKind() == MCFragment::FT_Relaxable)
+ Changed |= padInstructionEncoding(*F, Asm.getEmitter(), Size);
+ if (F == BF->getLastFragment() || Size == 0)
+ break;
+ }
+ };
+
+ unsigned TailSize = EndOffset % Asm.getBundleAlignSize();
+ if (!CrossBoundary && RemainingSize > 0 && TailSize != 0) {
+ padInstsForward(RemainingSize);
+ } else if (CrossBoundary && TailSize > 0) {
+ unsigned NextRemainingSize = TailSize;
+ padInstsForward(NextRemainingSize);
+ RemainingSize += NextRemainingSize;
+ }
+
+ // FT_Align sizes will be recalculated by layoutSection(),
+ // FT_BoundaryAlign sizes are adjusted here.
+ if (auto *BF = dyn_cast<MCBoundaryAlignFragment>(LastF))
+ BF->setSize(RemainingSize);
+
+ return Changed;
+}
+
+bool X86AsmBackend::optimizeBundleNops(const MCAssembler &Asm) const {
+ bool Changed = false;
+ for (MCSection &Sec : Asm) {
+ if (!Sec.isText())
+ continue;
+
+ SmallVector<MCFragment *, 4> Bundle;
+ for (MCSection::iterator I = Sec.begin(), IE = Sec.end(); I != IE; ++I) {
+ MCFragment &F = *I;
+
+ if (F.getKind() == llvm::MCFragment::FT_BoundaryAlign) {
+ unsigned RemainingSize = Asm.computeFragmentSize(F) - F.getFixedSize();
+ if (RemainingSize > 0) {
+ Bundle.push_back(&F);
+ Changed |= dividePadInBundle(Asm, Bundle);
+ Bundle.clear();
+ continue;
+ }
+ }
+
+ if (Asm.getFragmentOffset(F) % Asm.getBundleAlignSize() == 0)
+ Bundle.clear(); // start a new bundle
+ Bundle.push_back(&F);
+ }
+ }
+
+ return Changed;
+}
+
bool X86AsmBackend::finishLayout() const {
+ if (Asm->isBundlingEnabled() && TargetPrefixMax != 0)
+ return optimizeBundleNops(*Asm);
// See if we can further relax some instructions to cut down on the number of
// nop bytes required for code alignment. The actual win is in reducing
// instruction count, not number of bytes. Modern X86-64 can easily end up
@@ -911,24 +1112,7 @@ bool X86AsmBackend::finishLayout() const {
// of the resulting code. If we later find a reason to expand
// particular instructions over others, we can adjust.
unsigned RemainingSize = Asm->computeFragmentSize(F) - F.getFixedSize();
- while (!Relaxable.empty() && RemainingSize != 0) {
- auto &RF = *Relaxable.pop_back_val();
- // Give the backend a chance to play any tricks it wishes to increase
- // the encoding size of the given instruction. Target independent code
- // will try further relaxation, but target's may play further tricks.
- Changed |= padInstructionEncoding(RF, Asm->getEmitter(), RemainingSize);
-
- // If we have an instruction which hasn't been fully relaxed, we can't
- // skip past it and insert bytes before it. Changing its starting
- // offset might require a larger negative offset than it can encode.
- // We don't need to worry about larger positive offsets as none of the
- // possible offsets between this and our align are visible, and the
- // ones afterwards aren't changing.
- if (mayNeedRelaxation(RF.getOpcode(), RF.getOperands(),
- *RF.getSubtargetInfo()))
- break;
- }
- Relaxable.clear();
+ padInstsBackward(Relaxable, RemainingSize);
// If we're looking at a boundary align, make sure we don't try to pad
// its target instructions for some following directive. Doing so would
diff --git a/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s b/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
new file mode 100644
index 0000000000000..fcd53711e656e
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/asm-printing-bundle-directives.s
@@ -0,0 +1,20 @@
+# RUN: llvm-mc -filetype=asm -triple x86_64 %s -o - 2>&1 | FileCheck %s
+
+## Just a simple test for the assembly emitter - making sure it emits back the
+## bundling directives.
+
+ .text
+foo:
+ .bundle_align_mode 4
+# CHECK: .bundle_align_mode 4
+ pushq %rbp
+ .bundle_lock
+# CHECK: .bundle_lock
+ cmpl %r14d, %ebp
+ jle .L_ELSE
+ .bundle_unlock
+# CHECK: .bundle_unlock
+ .bundle_lock align_to_end
+# CHECK: .bundle_lock align_to_end
+ add %rbx, %rdx
+ .bundle_unlock
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s b/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
new file mode 100644
index 0000000000000..41803b6f8d38c
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-after-relax.s
@@ -0,0 +1,53 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d - | FileCheck %s
+
+## Test that instructions inside bundle-locked groups are relaxed even if their
+## fixup is short enough not to warrant relaxation on its own.
+ .text
+relax_in_bundle:
+ .bundle_align_mode 4
+ pushq %rbp
+
+ movl %edi, %ebx
+ callq bar
+ movl %eax, %r14d
+ imull $17, %ebx, %ebp
+ movl %ebx, %edi
+ callq bar
+ cmpl %r14d, %ebp
+ .bundle_lock
+
+ jle .L_ELSE
+## This group would've started at 0x18 and is too long, so a chunky NOP padding
+## is inserted to push it to 0x20.
+# CHECK: 18: {{[a-f0-9 ]+}} nopl
+
+## The long encoding for JLE should be used here even though its target is close
+# CHECK-NEXT: 20: 0f 8e
+
+ addl %ebp, %eax
+
+ jmp .L_RET
+## Same for the JMP
+# CHECK: 28: e9
+
+ .bundle_unlock
+
+.L_ELSE:
+ imull %ebx, %eax
+.L_RET:
+
+ popq %rbx
+
+## Test that an instruction near a bundle end gets properly padded to the next
+## bundle after it is relaxed.
+ .align 16
+relax_at_bundle_end:
+ .rept 14
+ push %rax
+ .endr
+# CHECK: 4d: 50 pushq
+# CHECK-NEXT: 4e: {{[a-f0-9 ]+}} nop
+# CHECK-NEXT: 50: 0f 85
+ jne 0x100
+
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s b/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
new file mode 100644
index 0000000000000..ef3dddc2a4ce2
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-align-to-end-lock.s
@@ -0,0 +1,65 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test NOP padding for `align_to_end` bundle-locked groups.
+
+ .text
+foo:
+ .bundle_align_mode 4
+
+## Each of these callq instructions is 5 bytes long
+ callq bar
+ callq bar
+ .bundle_lock align_to_end
+ callq bar
+ .bundle_unlock
+## To align this group to a bundle end, we need a 1-byte NOP.
+# CHECK: a: nop
+# CHECK-NEXT: b: callq
+
+ callq bar
+ callq bar
+ .bundle_lock align_to_end
+ callq bar
+ callq bar
+ .bundle_unlock
+## Here we have to pad until the end of the *next* boundary because
+## otherwise the group crosses a boundary.
+# CHECK: 1a: nop
+## The nop sequence may be implemented as one instruction or many, but if
+## it's one instruction, that instruction cannot itself cross the boundary.
+# CHECK: 20: nop
+# CHECK-NEXT: 26: callq
+# CHECK-NEXT: 2b: callq
+
+ .align 16, 0x90
+lock_to_next_bundle:
+ .rept 14
+ inc %eax
+ .endr
+ .bundle_lock align_to_end
+ inc %eax
+ inc %eax
+ inc %eax
+ .bundle_unlock
+## This bundle group must be adjusted to the next boundary,
+## but nop optimization must not break the bundle too.
+# CHECK: 30: inc
+# CHECK: 4a: inc
+# CHECK-NEXT: 4c: nop
+# CHECK-NEXT: 50: nop
+# CHECK-NEXT: 5a: inc
+
+lock_fit_exactly:
+ .rept 14
+ inc %eax
+ .endr
+ .bundle_lock align_to_end
+ inc %eax
+ inc %eax
+ .bundle_unlock
+# CHECK: 60: inc
+# CHECK: 7e: inc
+
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-errors.s b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
new file mode 100644
index 0000000000000..70e9795b74363
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-errors.s
@@ -0,0 +1,78 @@
+# RUN: split-file %s %t
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/lock-without-mode.s 2>&1 | FileCheck %t/lock-without-mode.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/mode-without-arg.s 2>&1 | FileCheck %t/mode-without-arg.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/unlock-without-lock.s 2>&1 | FileCheck %t/unlock-without-lock.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/bad-lock-option.s 2>&1 | FileCheck %t/bad-lock-option.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/switch-section-locked.s 2>&1 | FileCheck %t/switch-section-locked.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/group-too-large.s 2>&1 | FileCheck %t/group-too-large.s
+# RUN: not llvm-mc -filetype=obj -triple x86_64 %t/nested-lock.s 2>&1 | FileCheck %t/nested-lock.s
+
+
+## .bundle_lock can't come without a .bundle_align_mode before it
+#--- lock-without-mode.s
+ imull $17, %ebx, %ebp
+# CHECK: [[#@LINE+1]]:3: error: .bundle_lock forbidden when bundling is disabled
+ .bundle_lock
+
+## .bundle_align_mode needs a following integer value
+#--- mode-without-arg.s
+# CHECK: [[#@LINE+1]]:21: error: unknown token in expression
+ .bundle_align_mode
+ imull $17, %ebx, %ebp
+
+## .bundle_unlock can't come without a .bundle_lock before it
+#--- unlock-without-lock.s
+ .bundle_align_mode 3
+ imull $17, %ebx, %ebp
+# CHECK: [[#@LINE+1]]:3: error: .bundle_unlock without matching lock
+ .bundle_unlock
+
+## .bundle_lock can only take one `align_to_end` flag or no flag.
+#--- bad-lock-option.s
+ .bundle_align_mode 4
+# CHECK: [[#@LINE+1]]:16: error: invalid option for `.bundle_lock`
+ .bundle_lock 5
+ imull $17, %ebx, %ebp
+ .bundle_unlock
+
+## This test invokes .bundle_lock and then switches to a different section
+## w/o the appropriate unlock.
+#--- switch-section-locked.s
+ .bundle_align_mode 3
+ .section text1, "x"
+ imull $17, %ebx, %ebp
+ .bundle_lock
+ imull $17, %ebx, %ebp
+
+# CHECK: [[#@LINE+1]]:3: error: unterminated .bundle_lock
+ .section text2, "x"
+ imull $17, %ebx, %ebp
+
+## bundle lock size cannot be bigger than the align mode size
+#--- group-too-large.s
+ .text
+foo:
+ .bundle_align_mode 4
+ pushq %rbp
+
+ .bundle_lock
+ pushq %r14
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+# CHECK: [[#@LINE+1]]:3: error: fragment can't be larger than a bundle size
+ .bundle_unlock
+
+## test that nested lock is emitting the right error.
+#--- nested-lock.s
+ .bundle_align_mode 4
+foo:
+## bundle alignment mode can be set more than once.
+ .bundle_align_mode 4
+ .bundle_lock
+# CHECK: [[#@LINE+1]]:3: error: nested .bundle_lock is not allowed
+ .bundle_lock
+ .bundle_unlock
+ .bundle_unlock
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-inst.s b/llvm/test/MC/X86/AlignedBundling/bundle-inst.s
new file mode 100644
index 0000000000000..ebe7650156ef3
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-inst.s
@@ -0,0 +1,36 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test NOP padding to remove every instruction that crosses a bundle boundary.
+
+ .text
+foo:
+ .bundle_align_mode 5
+ ## 5 bytes * 6 = 30 bytes
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+
+## This imull is 3 bytes long and should have started at 0x1d, so two bytes
+## of nop padding are inserted instead and it starts at 0x20
+ imull $17, %ebx, %ebp
+# CHECK: 1e: nop
+# CHECK-NEXT: 20: imull
+
+## Sub-bundle .align with single-instruction bundling:
+## .align 16 is narrower than the 32-byte bundle; instructions after it
+## start mid-bundle and still receive NOP padding as needed.
+ pushq %rbp
+ .align 16
+ movl $1, (%rsp) ## 7 bytes at offset 16 — no padding needed
+ movl $2, 4(%rsp) ## 8 bytes; offset 23 → 23+8=31, fits in bundle
+ movl $3, (%rsp) ## 7 bytes at offset 31 → crosses boundary, pad to 32
+# CHECK: 30: mov
+# CHECK-NEXT: 37: mov
+# CHECK-NEXT: 3f: nop
+# CHECK-NEXT: 40: mov
diff --git a/llvm/test/MC/X86/AlignedBundling/bundle-lock.s b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
new file mode 100644
index 0000000000000..fca55509e3f7a
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/bundle-lock.s
@@ -0,0 +1,57 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test NOP padding for bundle-locked groups.
+
+ .text
+foo:
+ .bundle_align_mode 4
+
+## Each of these callq instructions is 5 bytes long
+ callq bar
+ callq bar
+
+ .bundle_lock
+ callq bar
+ callq bar
+ .bundle_unlock
+## We'll need a 6-byte NOP before this group
+# CHECK: a: nop
+# CHECK-NEXT: 10: callq
+# CHECK-NEXT: 15: callq
+
+ .bundle_lock
+ callq bar
+ callq bar
+ .bundle_unlock
+## Same here
+# CHECK: 1a: nop
+# CHECK-NEXT: 20: callq
+# CHECK-NEXT: 25: callq
+
+ .align 16, 0x90
+lock_to_next_bundle:
+ callq bar
+ .bundle_lock
+ callq bar
+ callq bar
+ callq bar
+ .bundle_unlock
+## And here we'll need a 10-byte NOP + 1-byte NOP
+# CHECK: 30: callq
+# CHECK: 35: nop
+# CHECK-NEXT: 40: callq
+# CHECK-NEXT: 45: callq
+
+ .align 16, 0x90
+lock_fit_exactly:
+## offset=6, group=10 bytes (5 x 2): 6+10=16 == BUNDLE_SIZE
+ .fill 6, 1, 0x90
+ .bundle_lock
+ .rept 5
+ inc %eax
+ .endr
+ .bundle_unlock
+# CHECK: 56: incl
diff --git a/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
new file mode 100644
index 0000000000000..be13274dc8a9b
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/long-nop-pad.s
@@ -0,0 +1,40 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 --mattr=+fast-15bytenop %s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+## Test that long nops are generated for padding where possible while each respects the bundle align boundary.
+
+ .text
+foo:
+ .bundle_align_mode 5
+
+## This callq instruction is 5 bytes long
+ .bundle_lock align_to_end
+ callq bar
+ .bundle_unlock
+## To align this group to a bundle end, we need a 15-byte NOPs and a 12-byte NOP.
+# CHECK: 0: nop
+# CHECK-NEXT: f: nop
+# CHECK-NEXT: 1b: callq
+
+## This push instruction is 1 byte long
+ .bundle_lock align_to_end
+ push %rax
+ .bundle_unlock
+## To align this group to a bundle end, we need two 15-byte NOPs and a 1-byte NOP.
+# CHECK: 20: nop
+# CHECK-NEXT: 2f: nop
+# CHECK-NEXT: 3e: nop
+# CHECK-NEXT: 3f: pushq
+
+## bundle-aware optimization for `.nops N`
+ .p2align 5
+just_nops:
+ callq bar
+ .nops 64
+# CHECK: 40: callq
+# CHECK-NEXT: 45: nop
+# CHECK-NEXT: 54: nop
+# CHECK-NEXT: 60: nop
+# CHECK-NEXT: 6f: nop
+# CHECK-NEXT: 7e: nop
+# CHECK-NEXT: 80: nop
diff --git a/llvm/test/MC/X86/AlignedBundling/prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
new file mode 100644
index 0000000000000..b0d6d60a08642
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/prefix-padding.s
@@ -0,0 +1,69 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o - --x86-pad-max-prefix-size=5 \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+ .text
+add_prefix_prev:
+ .bundle_align_mode 5
+## This callq instruction is 5 bytes long
+ callq bar
+ callq bar
+ callq bar
+ callq bar
+ .bundle_lock align_to_end
+ callq bar
+ .bundle_unlock
+# CHECK: 0: call
+# CHECK-NEXT: 5: call
+# CHECK-NEXT: a: call
+# CHECK-NEXT: 11: call
+# CHECK-NEXT: 1b: call
+
+ .p2align 5
+add_prefix_prev_next:
+ callq bar
+ .bundle_lock align_to_end
+ ## instructions inside a bundle lock can also be prefix-padded.
+ callq bar
+ .bundle_unlock
+# CHECK: 20: call
+# CHECK-NEXT: 2a: nop
+# CHECK: 36: call
+
+ .p2align 5
+ignore_nop_for_p2align:
+ int3
+ int3
+ ## no prefix padding with this 14-byte nop.
+ .p2align 4
+ int3
+ .bundle_lock
+ int3
+ .bundle_unlock
+ int3
+# CHECK: 40: int3
+# CHECK-NEXT: 41: int3
+# CHECK-NEXT: 42: nop
+# CHECK: 50: int3
+# CHECK-NEXT: 51: int3
+# CHECK-NEXT: 52: int3
+# CHECK-NEXT: 53: nop
+
+ .p2align 5
+ignore_nop_for_p2align5:
+ callq bar
+ .p2align 5
+.L1:
+ callq bar
+# CHECK: 60: call
+# CHECK-NEXT: 65: nop
+# CHECK: 80: call
+
+## ensure the last instructions are not prefix-padded
+ .p2align 5
+tail_bundle:
+ .bundle_lock
+ callq bar
+ .bundle_unlock
+ nop
+# CHECK: a0: call
+# CHECK-NEXT: a5: nop
diff --git a/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s b/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
new file mode 100644
index 0000000000000..5161bf4912c0b
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/relax-for-prefix-padding.s
@@ -0,0 +1,48 @@
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o - --x86-pad-max-prefix-size=1 \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %s
+
+# This test checks whether enabling prefix padding with bundling can properly relax instructions proactively, avoiding fixup value overflows.
+
+ .text
+ .bundle_align_mode 5
+ .p2align 5
+
+ .rept 3
+ callq x ## 5 bytes each
+ .endr
+# CHECK: f: jmp
+# CHECK-NEXT: 14: int3
+ jmp near_target ## 2 bytes (rel8), has to be relaxed to 5 bytes (rel32)
+ .rept 8
+ int3 ## 1 byte each, 7 of them are prefix-padded
+ .endr
+ ## trailing NOPs are only consumed by prefix-padding the int3 instructions
+# CHECK: 1e: int3
+# CHECK-NEXT: 20: int3
+
+ ## Three full bundles of spacer to push near_target close to rel8 max range
+ .rept 3
+ .bundle_lock
+ .rept 32
+ int3
+ .endr
+ .bundle_unlock
+ .endr
+
+ .rept 15
+ int3
+ .endr
+ ## With prefix padding (max 1): instructions absorb trailing NOPs,
+ ## near_target shifts from 0x8f to 0x9e.
+ ## jmp distance = 0x9d - (0xf + len(jmp)) = 0x8c (140), exceeds rel8 range,
+ ## forcing relaxation to rel32.
+# CHECK: <near_target>:
+# CHECK-NEXT: 9d: inc
+near_target:
+ inc %eax
+
+ .bundle_lock
+ .rept 32
+ int3
+ .endr
+ .bundle_unlock
diff --git a/llvm/test/MC/X86/AlignedBundling/section.s b/llvm/test/MC/X86/AlignedBundling/section.s
new file mode 100644
index 0000000000000..fcd636cf2528d
--- /dev/null
+++ b/llvm/test/MC/X86/AlignedBundling/section.s
@@ -0,0 +1,49 @@
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/two-sections.s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/two-sections.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 -mc-relax-all %t/two-sections.s -o - \
+# RUN: | llvm-objdump -d --no-show-raw-insn - | FileCheck %t/two-sections.s
+# RUN: llvm-mc -filetype=obj -triple x86_64 %t/section-alignment.s -o - \
+# RUN: | llvm-readobj --sections - | FileCheck %t/section-alignment.s
+
+## Test two different executable sections with bundling.
+#--- two-sections.s
+ .bundle_align_mode 3
+ .section text1, "x"
+# CHECK: section text1
+ imull $17, %ebx, %ebp
+ imull $17, %ebx, %ebp
+
+ imull $17, %ebx, %ebp
+# CHECK: 6: nop
+# CHECK-NEXT: 8: imull
+
+ .section text2, "x"
+# CHECK: section text2
+ imull $17, %ebx, %ebp
+ imull $17, %ebx, %ebp
+
+ imull $17, %ebx, %ebp
+# CHECK: 6: nop
+# CHECK-NEXT: 8: imull
+
+## Test that bundle-aligned sections with instructions are aligned
+#--- section-alignment.s
+ .bundle_align_mode 5
+# CHECK: Sections
+## Check that the empty .text section has the default alignment
+# CHECK-LABEL: Name: .text
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 4
+
+ .section text1, "x"
+ imull $17, %ebx, %ebp
+# CHECK-LABEL: Name: text1
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 32
+
+ .section text2, "x"
+ imull $17, %ebx, %ebp
+# CHECK-LABEL: Name: text2
+# CHECK-NOT: Name
+# CHECK: AddressAlignment: 32
More information about the llvm-commits
mailing list