[llvm] [BOLT] Create .text.warm for 3-way splitting (PR #73863)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 29 17:00:27 PST 2023
https://github.com/ShatianWang updated https://github.com/llvm/llvm-project/pull/73863
>From 062eee7a05e25723416a4989ed7929b8ba522752 Mon Sep 17 00:00:00 2001
From: Shatian Wang <shatian at meta.com>
Date: Mon, 27 Nov 2023 19:45:55 -0800
Subject: [PATCH 1/2] [BOLT] Create .text.warm for 3-way splitting
Summary:
This commit explicitly adds a warm code section, .text.warm, when the
-split-functions -split-strategy=cdsplit is used. This replaces the
previous approach of using .text.cold.0 as warm and .text.cold.1 as
cold in 3-way function splitting. NFC.
---
bolt/include/bolt/Core/BinaryContext.h | 7 +++++
bolt/include/bolt/Core/BinaryFunction.h | 2 ++
bolt/include/bolt/Core/FunctionLayout.h | 1 +
bolt/lib/Core/BinaryEmitter.cpp | 5 +++-
bolt/lib/Passes/SplitFunctions.cpp | 12 ++++----
bolt/lib/Rewrite/RewriteInstance.cpp | 39 ++++++++++++++++++-------
6 files changed, 49 insertions(+), 17 deletions(-)
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 48a8b2a5d83f2b0..c7672285b06c09f 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -614,6 +614,11 @@ class BinaryContext {
/// Indicates if the function ordering of the binary is finalized.
bool HasFinalizedFunctionOrder{false};
+ /// Indicates if a separate .text.warm section is needed that contains
+ /// function fragments with
+ /// FunctionFragment::getFragmentNum() == FragmentNum::warm()
+ bool HasWarmSection{false};
+
/// Is the binary always loaded at a fixed address. Shared objects and
/// position-independent executables (PIEs) are examples of binaries that
/// will have HasFixedLoadAddress set to false.
@@ -930,6 +935,8 @@ class BinaryContext {
const char *getMainCodeSectionName() const { return ".text"; }
+ const char *getWarmCodeSectionName() const { return ".text.warm"; }
+
const char *getColdCodeSectionName() const { return ".text.cold"; }
const char *getHotTextMoverSectionName() const { return ".text.mover"; }
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index 72c360ca0c2db66..182d5ff049c3762 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -1236,6 +1236,8 @@ class BinaryFunction {
return SmallString<32>(CodeSectionName);
if (Fragment == FragmentNum::cold())
return SmallString<32>(ColdCodeSectionName);
+ if (BC.HasWarmSection && Fragment == FragmentNum::warm())
+ return SmallString<32>(BC.getWarmCodeSectionName());
return formatv("{0}.{1}", ColdCodeSectionName, Fragment.get() - 1);
}
diff --git a/bolt/include/bolt/Core/FunctionLayout.h b/bolt/include/bolt/Core/FunctionLayout.h
index 904da3a4a93aade..2e4c184ba4511ca 100644
--- a/bolt/include/bolt/Core/FunctionLayout.h
+++ b/bolt/include/bolt/Core/FunctionLayout.h
@@ -63,6 +63,7 @@ class FragmentNum {
static constexpr FragmentNum main() { return FragmentNum(0); }
static constexpr FragmentNum cold() { return FragmentNum(1); }
+ static constexpr FragmentNum warm() { return FragmentNum(2); }
};
/// A freestanding subset of contiguous blocks of a function.
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index fb1bf530c1974aa..3bff3125a57a860 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -287,7 +287,10 @@ void BinaryEmitter::emitFunctions() {
// Mark the end of hot text.
if (opts::HotText) {
- Streamer.switchSection(BC.getTextSection());
+ if (BC.HasWarmSection)
+ Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName()));
+ else
+ Streamer.switchSection(BC.getTextSection());
Streamer.emitLabel(BC.getHotTextEndSymbol());
}
}
diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp
index af2a5aaa27ed5f4..79da4cc3b04ba95 100644
--- a/bolt/lib/Passes/SplitFunctions.cpp
+++ b/bolt/lib/Passes/SplitFunctions.cpp
@@ -160,16 +160,15 @@ struct SplitCacheDirected final : public SplitStrategy {
// Assign fragments based on the computed best split index.
// All basic blocks with index up to the best split index become hot.
// All remaining blocks are warm / cold depending on if count is
- // greater than 0 or not.
- FragmentNum Main(0);
- FragmentNum Cold(1);
- FragmentNum Warm(2);
+ // greater than zero or not.
for (size_t Index = 0; Index < BlockOrder.size(); Index++) {
BinaryBasicBlock *BB = BlockOrder[Index];
if (Index <= BestSplitIndex)
- BB->setFragmentNum(Main);
+ BB->setFragmentNum(FragmentNum::main());
else
- BB->setFragmentNum(BB->getKnownExecutionCount() > 0 ? Warm : Cold);
+ BB->setFragmentNum(BB->getKnownExecutionCount() > 0
+ ? FragmentNum::warm()
+ : FragmentNum::cold());
}
}
@@ -313,6 +312,7 @@ void SplitFunctions::runOnFunctions(BinaryContext &BC) {
else
Strategy = std::make_unique<SplitProfile2>();
opts::AggressiveSplitting = true;
+ BC.HasWarmSection = true;
break;
case SplitFunctionsStrategy::Profile2:
Strategy = std::make_unique<SplitProfile2>();
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 81c9cbff726bb9a..75904cf7d39d8b0 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -3478,11 +3478,21 @@ std::vector<BinarySection *> RewriteInstance::getCodeSections() {
if (B->getName() == BC->getHotTextMoverSectionName())
return false;
- // Depending on the option, put main text at the beginning or at the end.
- if (opts::HotFunctionsAtEnd)
- return B->getName() == BC->getMainCodeSectionName();
- else
- return A->getName() == BC->getMainCodeSectionName();
+ // Depending on opts::HotFunctionsAtEnd, place main and warm sections in
+ // order.
+ if (opts::HotFunctionsAtEnd) {
+ if (B->getName() == BC->getMainCodeSectionName())
+ return true;
+ if (A->getName() == BC->getMainCodeSectionName())
+ return false;
+ return (B->getName() == BC->getWarmCodeSectionName());
+ } else {
+ if (A->getName() == BC->getMainCodeSectionName())
+ return true;
+ if (B->getName() == BC->getMainCodeSectionName())
+ return false;
+ return (A->getName() == BC->getWarmCodeSectionName());
+ }
};
// Determine the order of sections.
@@ -3524,6 +3534,9 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
// Allocate sections starting at a given Address.
auto allocateAt = [&](uint64_t Address) {
+ const char *LastNonColdSectionName = BC->HasWarmSection
+ ? BC->getWarmCodeSectionName()
+ : BC->getMainCodeSectionName();
for (BinarySection *Section : CodeSections) {
Address = alignTo(Address, Section->getAlignment());
Section->setOutputAddress(Address);
@@ -3532,13 +3545,13 @@ void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
// Hugify: Additional huge page from right side due to
// weird ASLR mapping addresses (4KB aligned)
if (opts::Hugify && !BC->HasFixedLoadAddress &&
- Section->getName() == BC->getMainCodeSectionName())
+ Section->getName() == LastNonColdSectionName)
Address = alignTo(Address, Section->getAlignment());
}
// Make sure we allocate enough space for huge pages.
ErrorOr<BinarySection &> TextSection =
- BC->getUniqueSectionByName(BC->getMainCodeSectionName());
+ BC->getUniqueSectionByName(LastNonColdSectionName);
if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
uint64_t HotTextEnd =
TextSection->getOutputAddress() + TextSection->getOutputSize();
@@ -4423,9 +4436,15 @@ void RewriteInstance::updateELFSymbolTable(
Function.getLayout().getSplitFragments()) {
if (FF.getAddress()) {
ELFSymTy NewColdSym = FunctionSymbol;
- const SmallString<256> SymbolName = formatv(
- "{0}.cold.{1}", cantFail(FunctionSymbol.getName(StringSection)),
- FF.getFragmentNum().get() - 1);
+ SmallString<256> SymbolName;
+ if (BC->HasWarmSection)
+ SymbolName = formatv(
+ "{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)),
+ FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold");
+ else
+ SymbolName = formatv(
+ "{0}.cold.{1}", cantFail(FunctionSymbol.getName(StringSection)),
+ FF.getFragmentNum().get() - 1);
NewColdSym.st_name = AddToStrTab(SymbolName);
NewColdSym.st_shndx =
Function.getCodeSection(FF.getFragmentNum())->getIndex();
>From f6e7cc092449bcd32513fe3e54ea72bf15fad472 Mon Sep 17 00:00:00 2001
From: Shatian Wang <shatian at meta.com>
Date: Wed, 29 Nov 2023 16:59:28 -0800
Subject: [PATCH 2/2] fixup! [BOLT] Create .text.warm for 3-way splitting
---
bolt/lib/Rewrite/RewriteInstance.cpp | 26 +++++++++++++++++---------
1 file changed, 17 insertions(+), 9 deletions(-)
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 75904cf7d39d8b0..8cda0b7fcca9f84 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -4409,6 +4409,21 @@ void RewriteInstance::updateELFSymbolTable(
return NewIndex;
};
+ // Get the extra symbol name of a split fragment; used in addExtraSymbols.
+ auto getSplitSymbolName = [&](const FunctionFragment &FF,
+ const ELFSymTy &FunctionSymbol) {
+ SmallString<256> SymbolName;
+ if (BC->HasWarmSection)
+ SymbolName =
+ formatv("{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)),
+ FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold");
+ else
+ SymbolName = formatv("{0}.cold.{1}",
+ cantFail(FunctionSymbol.getName(StringSection)),
+ FF.getFragmentNum().get() - 1);
+ return SymbolName;
+ };
+
// Add extra symbols for the function.
//
// Note that addExtraSymbols() could be called multiple times for the same
@@ -4436,15 +4451,8 @@ void RewriteInstance::updateELFSymbolTable(
Function.getLayout().getSplitFragments()) {
if (FF.getAddress()) {
ELFSymTy NewColdSym = FunctionSymbol;
- SmallString<256> SymbolName;
- if (BC->HasWarmSection)
- SymbolName = formatv(
- "{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)),
- FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold");
- else
- SymbolName = formatv(
- "{0}.cold.{1}", cantFail(FunctionSymbol.getName(StringSection)),
- FF.getFragmentNum().get() - 1);
+ const SmallString<256> SymbolName =
+ getSplitSymbolName(FF, FunctionSymbol);
NewColdSym.st_name = AddToStrTab(SymbolName);
NewColdSym.st_shndx =
Function.getCodeSection(FF.getFragmentNum())->getIndex();
More information about the llvm-commits
mailing list