[llvm] [MachineOutliner] Remove LOHs from outlined candidates (PR #143617)
Ellis Hoag via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 11 10:11:24 PDT 2025
https://github.com/ellishg updated https://github.com/llvm/llvm-project/pull/143617
>From 30acec2164a34bbb9a8d6cf12ffd52efe480dbc4 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Tue, 10 Jun 2025 10:34:35 -0700
Subject: [PATCH 1/2] [LOH] Remove hints when outlining
---
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 6 ++
llvm/lib/CodeGen/MachineOutliner.cpp | 18 ++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 16 ++---
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 1 +
.../AArch64/AArch64MachineFunctionInfo.h | 13 ++++
.../CodeGen/AArch64/machine-outliner-loh.ll | 63 +++++++++++++++++++
6 files changed, 110 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index b5b83c7ff1164..3aec8791b7957 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2184,6 +2184,12 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
MachineBasicBlock::iterator &MIT,
unsigned Flags) const;
+ /// Remove all Linker Optimization Hints associated with instructions in
+ // \p MIs and \return the number of hints removed.
+ virtual size_t clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+ return 0;
+ }
+
/// Optional target hook that returns true if \p MBB is safe to outline from,
/// and returns any target-specific information in \p Flags.
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index e48612369a5db..02a08e4d50818 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -104,6 +104,7 @@ STATISTIC(StableHashAttempts,
"Count of hashing attempts made for outlined functions");
STATISTIC(StableHashDropped,
"Count of unsuccessful hashing attempts for outlined functions");
+STATISTIC(NumRemovedLOHs, "Total number of Linker Optimization Hints removed");
// Set to true if the user wants the outliner to run on linkonceodr linkage
// functions. This is false by default because the linker can dedupe linkonceodr
@@ -1075,6 +1076,23 @@ bool MachineOutliner::outline(
<< " B) > threshold (" << OutlinerBenefitThreshold
<< " B)\n");
+ // Remove all Linker Optimization Hints from the candidates since we did not
+ // check if the set of hints are the same for each of them.
+ // TODO: The intersection of the LOHs from all candidates should be legal in
+ // the outlined function.
+ SmallPtrSet<MachineInstr *, 2> MIs;
+ std::optional<size_t> MinRemovedLOHs;
+ for (Candidate &C : OF->Candidates) {
+ const TargetInstrInfo &TII = *C.getMF()->getSubtarget().getInstrInfo();
+ for (MachineInstr &MI : C)
+ MIs.insert(&MI);
+ size_t NumRemoved = TII.clearLOHs(MIs);
+ MIs.clear();
+ MinRemovedLOHs =
+ std::min(MinRemovedLOHs.value_or(NumRemoved), NumRemoved);
+ }
+ NumRemovedLOHs += MinRemovedLOHs.value_or(0);
+
// It's beneficial. Create the function and outline its sequence's
// occurrences.
OF->MF = createOutlinedFunction(M, *OF, Mapper, OutlinedFunctionNum);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 951cb93ea8f8c..ea04bd4cf2428 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -9641,14 +9641,20 @@ AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
return Ranges;
}
+size_t
+AArch64InstrInfo::clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+ if (MIs.empty())
+ return 0;
+ auto *MI = *MIs.begin();
+ auto *FuncInfo = MI->getMF()->getInfo<AArch64FunctionInfo>();
+ return FuncInfo->clearLOHs(MIs);
+}
+
outliner::InstrType
AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
MachineBasicBlock::iterator &MIT,
unsigned Flags) const {
MachineInstr &MI = *MIT;
- MachineBasicBlock *MBB = MI.getParent();
- MachineFunction *MF = MBB->getParent();
- AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
// Don't outline anything used for return address signing. The outlined
// function will get signed later if needed
@@ -9676,10 +9682,6 @@ AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
return outliner::InstrType::Illegal;
}
- // Don't outline LOHs.
- if (FuncInfo->getLOHRelated().count(&MI))
- return outliner::InstrType::Illegal;
-
// We can only outline these if we will tail call the outlined function, or
// fix up the CFI offsets. Currently, CFI instructions are outlined only if
// in a tail call.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 7c255da333e4b..45817e85e3826 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -493,6 +493,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,
MachineBasicBlock::iterator &MIT,
unsigned Flags) const override;
+ size_t clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) const override;
SmallVector<
std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override;
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 361d5ec3f2b22..0c0217cf4fa3f 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -504,6 +504,19 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
LOHRelated.insert_range(Args);
}
+ size_t clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) {
+ size_t InitialSize = LOHContainerSet.size();
+ erase_if(LOHContainerSet, [&](const auto &D) {
+ return any_of(D.getArgs(), [&](auto *Arg) { return MIs.contains(Arg); });
+ });
+ // In theory there could be an LOH with one label in MIs and another label
+ // outside MIs, however we don't know if the label outside MIs is used in
+ // any other LOHs, so we can't remove them from LOHRelated. In that case, we
+ // might produce a few extra labels, but it won't break anything.
+ LOHRelated.remove_if([&](auto *MI) { return MIs.contains(MI); });
+ return InitialSize - LOHContainerSet.size();
+ };
+
SmallVectorImpl<ForwardedRegister> &getForwardedMustTailRegParms() {
return ForwardedMustTailRegParms;
}
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll b/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
new file mode 100644
index 0000000000000..66914d70c23f9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
@@ -0,0 +1,63 @@
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-darwin < %s | FileCheck %s --implicit-check-not=.loh --check-prefixes=CHECK,LOH
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-darwin -enable-machine-outliner < %s | FileCheck %s --implicit-check-not=.loh --check-prefixes=CHECK,OUTLINE
+
+ at A = global i32 0, align 4
+ at B = global i32 0, align 4
+
+declare void @foo();
+declare void @bar(ptr %a);
+declare void @goo(ptr %a);
+
+; CHECK-LABEL: _a0:
+define i32 @a0(i32 %a) {
+ %addr = getelementptr inbounds i32, ptr @A, i32 0
+ %res = load i32, ptr %addr, align 4
+ ; LOH: [[L0:Lloh.+]]:
+ ; LOH-NEXT: adrp x19, _A at PAGE
+ ; LOH-NEXT: [[L1:Lloh.+]]:
+ ; LOH-NEXT: add x19, x19, _A at PAGEOFF
+ call void @foo()
+ ; OUTLINE: bl _OUTLINED_FUNCTION_0
+ ; OUTLINE-NEXT: mov x0, x19
+ ; OUTLINE-NEXT: bl _bar
+ call void @bar(ptr %addr)
+ %addr2 = getelementptr inbounds i32, ptr @B, i32 4
+ store i32 0, ptr %addr2, align 4
+ ; CHECK: [[L2:Lloh.+]]:
+ ; CHECK-NEXT: adrp x8, _B at PAGE
+ ; CHECK-NEXT: [[L3:Lloh.+]]:
+ ; CHECK-NEXT: add x8, x8, _B at PAGEOFF
+ ; CHECK-NEXT: mov w0, w20
+ ; CHECK-NEXT: [[L4:Lloh.+]]:
+ ; CHECK-NEXT: str wzr, [x8, #16]
+ ret i32 %res
+ ; LOH-DAG: .loh AdrpAdd [[L0]], [[L1]]
+ ; CHECK-DAG: .loh AdrpAddStr [[L2]], [[L3]], [[L4]]
+ ; CHECK: .cfi_endproc
+}
+
+; CHECK-LABEL: _a1:
+define i32 @a1(i32 %a) {
+ %addr = getelementptr inbounds i32, ptr @A, i32 0
+ %res = load i32, ptr %addr, align 4
+ ; LOH: [[L5:Lloh.+]]:
+ ; LOH-NEXT: adrp x19, _A at PAGE
+ ; LOH-NEXT: [[L6:Lloh.+]]:
+ ; LOH-NEXT: add x19, x19, _A at PAGEOFF
+ call void @foo()
+ ; OUTLINE: bl _OUTLINED_FUNCTION_0
+ ; OUTLINE-NEXT: mov x0, x19
+ ; OUTLINE-NEXT: bl _goo
+ call void @goo(ptr %addr)
+ ret i32 %res
+ ; LOH: .loh AdrpAdd [[L5]], [[L6]]
+ ; CHECK: .cfi_endproc
+}
+
+; Note: it is not safe to add LOHs to this function as outlined functions do not
+; follow calling convention and thus x19 could be live across the call.
+; OUTLINE: _OUTLINED_FUNCTION_0:
+; OUTLINE: adrp x19, _A at PAGE
+; OUTLINE: add x19, x19, _A at PAGEOFF
+; OUTLINE: ldr w20, [x19]
+; OUTLINE: b _foo
>From 09aeee933db51e1176a8f3f36cd781aac1c8d4c6 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellishoag at meta.com>
Date: Wed, 11 Jun 2025 10:11:10 -0700
Subject: [PATCH 2/2] rename func
---
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 9 ++++++---
llvm/lib/CodeGen/MachineOutliner.cpp | 2 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 6 +++---
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 3 ++-
llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h | 3 ++-
5 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 3aec8791b7957..d32578540f207 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2184,9 +2184,12 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
MachineBasicBlock::iterator &MIT,
unsigned Flags) const;
- /// Remove all Linker Optimization Hints associated with instructions in
- // \p MIs and \return the number of hints removed.
- virtual size_t clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+ /// Remove all Linker Optimization Hints (LOH) associated with instructions in
+ /// \p MIs and \return the number of hints removed. This is useful in
+ /// transformations that cause these hints to be illegal, like in the machine
+ /// outliner.
+ virtual size_t clearLinkerOptimizationHints(
+ const SmallPtrSetImpl<MachineInstr *> &MIs) const {
return 0;
}
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index 02a08e4d50818..9934482673b4a 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -1086,7 +1086,7 @@ bool MachineOutliner::outline(
const TargetInstrInfo &TII = *C.getMF()->getSubtarget().getInstrInfo();
for (MachineInstr &MI : C)
MIs.insert(&MI);
- size_t NumRemoved = TII.clearLOHs(MIs);
+ size_t NumRemoved = TII.clearLinkerOptimizationHints(MIs);
MIs.clear();
MinRemovedLOHs =
std::min(MinRemovedLOHs.value_or(NumRemoved), NumRemoved);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index ea04bd4cf2428..c3e23a1bdecc8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -9641,13 +9641,13 @@ AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
return Ranges;
}
-size_t
-AArch64InstrInfo::clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+size_t AArch64InstrInfo::clearLinkerOptimizationHints(
+ const SmallPtrSetImpl<MachineInstr *> &MIs) const {
if (MIs.empty())
return 0;
auto *MI = *MIs.begin();
auto *FuncInfo = MI->getMF()->getInfo<AArch64FunctionInfo>();
- return FuncInfo->clearLOHs(MIs);
+ return FuncInfo->clearLinkerOptimizationHints(MIs);
}
outliner::InstrType
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 45817e85e3826..34882c96c4497 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -493,7 +493,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,
MachineBasicBlock::iterator &MIT,
unsigned Flags) const override;
- size_t clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) const override;
+ size_t clearLinkerOptimizationHints(
+ const SmallPtrSetImpl<MachineInstr *> &MIs) const override;
SmallVector<
std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override;
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 0c0217cf4fa3f..e61f2280865b4 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -504,7 +504,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
LOHRelated.insert_range(Args);
}
- size_t clearLOHs(const SmallPtrSetImpl<MachineInstr *> &MIs) {
+ size_t
+ clearLinkerOptimizationHints(const SmallPtrSetImpl<MachineInstr *> &MIs) {
size_t InitialSize = LOHContainerSet.size();
erase_if(LOHContainerSet, [&](const auto &D) {
return any_of(D.getArgs(), [&](auto *Arg) { return MIs.contains(Arg); });
More information about the llvm-commits
mailing list