[llvm] [BOLT] BB can be cold, warm and hot. (PR #121475)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 22 05:52:32 PST 2025
https://github.com/liusy58 updated https://github.com/llvm/llvm-project/pull/121475
>From ad3a483661f6c552b90470571db9a6ed4bd4f984 Mon Sep 17 00:00:00 2001
From: liusy58 <liusy58 at linux.alibaba.com>
Date: Thu, 2 Jan 2025 20:53:27 +0800
Subject: [PATCH 1/2] [BOLT] BB can be cold, warm and hot.
CDSplit enables BB to be cold, warm or hot. So this assert should be removed.
---
bolt/include/bolt/Core/BinaryBasicBlock.h | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h
index 25cccc4edecf68..5818ed240f88c5 100644
--- a/bolt/include/bolt/Core/BinaryBasicBlock.h
+++ b/bolt/include/bolt/Core/BinaryBasicBlock.h
@@ -672,11 +672,11 @@ class BinaryBasicBlock {
bool isSplit() const { return Fragment != FragmentNum::main(); }
- bool isCold() const {
- assert(Fragment.get() < 2 &&
- "Function is split into more than two (hot/cold)-fragments");
- return isSplit();
- }
+ bool isCold() const { return Fragment == FragmentNum::cold(); }
+
+ bool isWarm() const { return Fragment == FragmentNum::warm(); }
+
+ bool isHot() const { return Fragment == FragmentNum::main(); }
void setIsCold(const bool Flag) {
Fragment = Flag ? FragmentNum::cold() : FragmentNum::main();
>From 26054fc1d0a383920432b447df92736bc2bfa49f Mon Sep 17 00:00:00 2001
From: liusy58 <liusy58 at linux.alibaba.com>
Date: Wed, 22 Jan 2025 21:34:42 +0800
Subject: [PATCH 2/2] [BOLT][AArch64] Support cdsplit for AArch64.
The current LongJmp pass only supports hot-cold splitting and is
unaware of the warm part. This oversight results in missing stubs
that need to be inserted. Add code for warm layout.
---
bolt/include/bolt/Core/BinaryBasicBlock.h | 2 +-
bolt/include/bolt/Passes/LongJmp.h | 11 +
bolt/lib/Passes/LongJmp.cpp | 299 ++++++++++++++++++----
bolt/test/AArch64/cdsplit-call-scale.s | 127 +++++++++
bolt/test/AArch64/cdsplit-symbol-names.s | 98 +++++++
5 files changed, 485 insertions(+), 52 deletions(-)
create mode 100644 bolt/test/AArch64/cdsplit-call-scale.s
create mode 100644 bolt/test/AArch64/cdsplit-symbol-names.s
diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h
index 5818ed240f88c5..3cf23a2c03dde2 100644
--- a/bolt/include/bolt/Core/BinaryBasicBlock.h
+++ b/bolt/include/bolt/Core/BinaryBasicBlock.h
@@ -676,7 +676,7 @@ class BinaryBasicBlock {
bool isWarm() const { return Fragment == FragmentNum::warm(); }
- bool isHot() const { return Fragment == FragmentNum::main(); }
+ bool isMain() const { return Fragment == FragmentNum::main(); }
void setIsCold(const bool Flag) {
Fragment = Flag ? FragmentNum::cold() : FragmentNum::main();
diff --git a/bolt/include/bolt/Passes/LongJmp.h b/bolt/include/bolt/Passes/LongJmp.h
index df3ea9620918af..2090ec6aa41d2a 100644
--- a/bolt/include/bolt/Passes/LongJmp.h
+++ b/bolt/include/bolt/Passes/LongJmp.h
@@ -37,6 +37,7 @@ class LongJmpPass : public BinaryFunctionPass {
using StubGroupsTy = DenseMap<const MCSymbol *, StubGroupTy>;
StubGroupsTy HotStubGroups;
StubGroupsTy ColdStubGroups;
+ StubGroupsTy WarmStubGroups;
DenseMap<const MCSymbol *, BinaryBasicBlock *> SharedStubs;
/// Stubs that are local to a function. This will be the primary lookup
@@ -45,6 +46,7 @@ class LongJmpPass : public BinaryFunctionPass {
/// Used to quickly fetch stubs based on the target they jump to
StubMapTy HotLocalStubs;
StubMapTy ColdLocalStubs;
+ StubMapTy WarmLocalStubs;
/// Used to quickly identify whether a BB is a stub, sharded by function
DenseMap<const BinaryFunction *, std::set<const BinaryBasicBlock *>> Stubs;
@@ -53,6 +55,7 @@ class LongJmpPass : public BinaryFunctionPass {
/// Hold tentative addresses
FuncAddressesMapTy HotAddresses;
FuncAddressesMapTy ColdAddresses;
+ FuncAddressesMapTy WarmAddresses;
DenseMap<const BinaryBasicBlock *, uint64_t> BBAddresses;
/// Used to identify the stub size
@@ -61,6 +64,7 @@ class LongJmpPass : public BinaryFunctionPass {
/// Stats about number of stubs inserted
uint32_t NumHotStubs{0};
uint32_t NumColdStubs{0};
+ uint32_t NumWarmStubs{0};
uint32_t NumSharedStubs{0};
/// The shortest distance for any branch instruction on AArch64.
@@ -91,6 +95,10 @@ class LongJmpPass : public BinaryFunctionPass {
tentativeLayoutRelocColdPart(const BinaryContext &BC,
std::vector<BinaryFunction *> &SortedFunctions,
uint64_t DotAddress);
+ uint64_t
+ tentativeLayoutRelocWarmPart(const BinaryContext &BC,
+ std::vector<BinaryFunction *> &SortedFunctions,
+ uint64_t DotAddress);
void tentativeBBLayout(const BinaryFunction &Func);
/// Update stubs addresses with their exact address after a round of stub
@@ -152,6 +160,9 @@ class LongJmpPass : public BinaryFunctionPass {
/// Relax function by adding necessary stubs or relaxing existing stubs
Error relax(BinaryFunction &BF, bool &Modified);
+ const StubGroupsTy &getStubGroupsTyByBB(const BinaryBasicBlock &BB) const;
+ const DenseMap<const BinaryFunction *, StubGroupsTy> &
+ getLocalStubGroupsTyByBB(const BinaryBasicBlock &BB) const;
public:
/// BinaryPass public interface
diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp
index e6bd417705e6ff..8b90a3db8382bb 100644
--- a/bolt/lib/Passes/LongJmp.cpp
+++ b/bolt/lib/Passes/LongJmp.cpp
@@ -41,6 +41,93 @@ namespace bolt {
constexpr unsigned ColdFragAlign = 16;
+static BinaryBasicBlock *getFrontier(BinaryFunction &Func,
+ BinaryBasicBlock &BB) {
+
+ // return the last bb of the current fragment since stub will be inserted
+ // just after this frontier
+ if (!Func.isSplit() || Func.empty())
+ return nullptr;
+ const FunctionFragment &Layout = Func.getLayout().findFragment(&BB);
+
+ for (auto I = Layout.begin(), E = Layout.end(); I != E; ++I) {
+ auto Next = std::next(I);
+ if (Next == E)
+ return *I;
+ }
+
+ llvm_unreachable("No split point found");
+}
+
+static const FunctionFragment *
+getWarmFragmentOrNullptr(const FunctionLayout &Layout) {
+ // See `bool FunctionLayout::update`
+ // FragmentNum Num = BB->getFragmentNum();
+ // while (Fragments.back()->getFragmentNum() < Num)
+ // addFragment();
+ // Fragments will be added according to FragmentNum, so
+ // we can use `FragmentNum` to get the fragment.
+
+ if (Layout.fragment_size() <= FragmentNum::warm().get()) {
+ return nullptr;
+ }
+ return &Layout.getFragment(FragmentNum::warm());
+}
+
+static const FunctionFragment *
+getColdFragmentOrNullptr(const FunctionLayout &Layout) {
+ if (Layout.fragment_size() <= FragmentNum::cold().get()) {
+ return nullptr;
+ }
+
+ return &Layout.getFragment(FragmentNum::cold());
+}
+
+static bool hasNonEmptyColdFragment(const BinaryFunction *BF) {
+ return BF->isSplit() && getColdFragmentOrNullptr(BF->getLayout()) &&
+ !getColdFragmentOrNullptr(BF->getLayout())->empty();
+}
+
+static bool hasNonEmptyWarmFragment(const BinaryFunction *BF) {
+ return BF->isSplit() && getWarmFragmentOrNullptr(BF->getLayout()) &&
+ !getWarmFragmentOrNullptr(BF->getLayout())->empty();
+}
+
+static size_t estimateWarmSize(const BinaryFunction *BF) {
+ const BinaryContext &BC = BF->getBinaryContext();
+ if (!BF->isSplit())
+ return BF->estimateSize();
+
+ size_t Estimate = 0;
+
+ for (const BinaryBasicBlock &BB : *BF)
+ if (BB.isWarm())
+ Estimate += BC.computeCodeSize(BB.begin(), BB.end());
+
+ return Estimate;
+}
+
+static size_t estimateHotSize(const BinaryFunction *BF,
+ const bool UseSplitSize = true) {
+ const BinaryContext &BC = BF->getBinaryContext();
+ size_t Estimate = 0;
+
+ if (UseSplitSize && BF->isSplit()) {
+ for (const BinaryBasicBlock &BB : *BF)
+ if (BB.isMain())
+ Estimate += BC.computeCodeSize(BB.begin(), BB.end());
+ } else {
+ for (const BinaryBasicBlock &BB : *BF)
+ if (BB.getKnownExecutionCount() != 0)
+ Estimate += BC.computeCodeSize(BB.begin(), BB.end());
+ }
+
+ return Estimate;
+}
+static size_t estimateSize(const BinaryFunction *BF) {
+ return BF->estimateSize();
+}
+
static void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
const BinaryContext &BC = StubBB.getFunction()->getBinaryContext();
InstructionListType Seq;
@@ -57,24 +144,68 @@ static void relaxStubToLongJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {
StubBB.addInstructions(Seq.begin(), Seq.end());
}
-static BinaryBasicBlock *getBBAtHotColdSplitPoint(BinaryFunction &Func) {
- if (!Func.isSplit() || Func.empty())
- return nullptr;
+static bool mayNeedStub(const BinaryContext &BC, const MCInst &Inst) {
+ return (BC.MIB->isBranch(Inst) || BC.MIB->isCall(Inst)) &&
+ !BC.MIB->isIndirectBranch(Inst) && !BC.MIB->isIndirectCall(Inst);
+}
- assert(!(*Func.begin()).isCold() && "Entry cannot be cold");
- for (auto I = Func.getLayout().block_begin(),
- E = Func.getLayout().block_end();
- I != E; ++I) {
- auto Next = std::next(I);
- if (Next != E && (*Next)->isCold())
- return *I;
+const LongJmpPass::StubGroupsTy &
+LongJmpPass::getStubGroupsTyByBB(const BinaryBasicBlock &BB) const {
+ if (BB.isMain()) {
+ return HotStubGroups;
+ }
+
+ if (BB.isWarm()) {
+ return WarmStubGroups;
}
- llvm_unreachable("No hot-cold split point found");
+
+ if (BB.isCold()) {
+ return ColdStubGroups;
+ }
+
+ llvm_unreachable("Should not reach here\n");
}
-static bool mayNeedStub(const BinaryContext &BC, const MCInst &Inst) {
- return (BC.MIB->isBranch(Inst) || BC.MIB->isCall(Inst)) &&
- !BC.MIB->isIndirectBranch(Inst) && !BC.MIB->isIndirectCall(Inst);
+const DenseMap<const BinaryFunction *, LongJmpPass::StubGroupsTy> &
+LongJmpPass::getLocalStubGroupsTyByBB(const BinaryBasicBlock &BB) const {
+ if (BB.isMain()) {
+ return HotLocalStubs;
+ }
+
+ if (BB.isWarm()) {
+ return WarmLocalStubs;
+ }
+
+ if (BB.isCold()) {
+ return ColdLocalStubs;
+ }
+
+ llvm_unreachable("Should not reach here\n");
+}
+
+uint64_t LongJmpPass::tentativeLayoutRelocWarmPart(
+ const BinaryContext &BC, std::vector<BinaryFunction *> &SortedFunctions,
+ uint64_t DotAddress) {
+ DotAddress = alignTo(DotAddress, llvm::Align(opts::AlignFunctions));
+ for (BinaryFunction *Func : SortedFunctions) {
+ if (!hasNonEmptyWarmFragment(Func))
+ continue;
+
+ DotAddress = alignTo(DotAddress, Func->getMinAlignment());
+ uint64_t Pad =
+ offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));
+ if (Pad <= Func->getMaxColdAlignmentBytes())
+ DotAddress += Pad;
+ // Warm
+ WarmAddresses[Func] = DotAddress;
+ LLVM_DEBUG(dbgs() << Func->getPrintName() << " warm tentative: "
+ << Twine::utohexstr(DotAddress) << "\n");
+ DotAddress += estimateWarmSize(Func);
+ DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());
+ DotAddress += Func->estimateConstantIslandSize();
+ }
+
+ return DotAddress;
}
std::pair<std::unique_ptr<BinaryBasicBlock>, MCSymbol *>
@@ -82,7 +213,6 @@ LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,
bool TgtIsFunc, uint64_t AtAddress) {
BinaryFunction &Func = *SourceBB.getFunction();
const BinaryContext &BC = Func.getBinaryContext();
- const bool IsCold = SourceBB.isCold();
MCSymbol *StubSym = BC.Ctx->createNamedTempSymbol("Stub");
std::unique_ptr<BinaryBasicBlock> StubBB = Func.createBasicBlock(StubSym);
MCInst Inst;
@@ -107,16 +237,23 @@ LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,
Stubs[&Func].insert(StubBB.get());
StubBits[StubBB.get()] = BC.MIB->getUncondBranchEncodingSize();
- if (IsCold) {
+ if (SourceBB.isCold()) {
registerInMap(ColdLocalStubs[&Func]);
if (opts::GroupStubs && TgtIsFunc)
registerInMap(ColdStubGroups);
++NumColdStubs;
- } else {
+ } else if (SourceBB.isWarm()) {
+ registerInMap(WarmLocalStubs[&Func]);
+ if (opts::GroupStubs && TgtIsFunc)
+ registerInMap(WarmStubGroups);
+ ++NumWarmStubs;
+ } else if (SourceBB.isMain()) {
registerInMap(HotLocalStubs[&Func]);
if (opts::GroupStubs && TgtIsFunc)
registerInMap(HotStubGroups);
++NumHotStubs;
+ } else {
+ llvm_unreachable("BB must be cold or warm or hot\n");
}
return std::make_pair(std::move(StubBB), StubSym);
@@ -168,8 +305,7 @@ LongJmpPass::lookupGlobalStub(const BinaryBasicBlock &SourceBB,
const MCInst &Inst, const MCSymbol *TgtSym,
uint64_t DotAddress) const {
const BinaryFunction &Func = *SourceBB.getFunction();
- const StubGroupsTy &StubGroups =
- SourceBB.isCold() ? ColdStubGroups : HotStubGroups;
+ const StubGroupsTy &StubGroups = getStubGroupsTyByBB(SourceBB);
return lookupStubFromGroup(StubGroups, Func, Inst, TgtSym, DotAddress);
}
@@ -179,7 +315,7 @@ BinaryBasicBlock *LongJmpPass::lookupLocalStub(const BinaryBasicBlock &SourceBB,
uint64_t DotAddress) const {
const BinaryFunction &Func = *SourceBB.getFunction();
const DenseMap<const BinaryFunction *, StubGroupsTy> &StubGroups =
- SourceBB.isCold() ? ColdLocalStubs : HotLocalStubs;
+ getLocalStubGroupsTyByBB(SourceBB);
const auto Iter = StubGroups.find(&Func);
if (Iter == StubGroups.end())
return nullptr;
@@ -287,23 +423,45 @@ void LongJmpPass::updateStubGroups() {
for (auto &KeyVal : ColdLocalStubs)
update(KeyVal.second);
update(HotStubGroups);
+ update(WarmStubGroups);
update(ColdStubGroups);
}
void LongJmpPass::tentativeBBLayout(const BinaryFunction &Func) {
const BinaryContext &BC = Func.getBinaryContext();
- uint64_t HotDot = HotAddresses[&Func];
- uint64_t ColdDot = ColdAddresses[&Func];
- bool Cold = false;
- for (const BinaryBasicBlock *BB : Func.getLayout().blocks()) {
- if (Cold || BB->isCold()) {
- Cold = true;
- BBAddresses[BB] = ColdDot;
- ColdDot += BC.computeCodeSize(BB->begin(), BB->end());
- } else {
- BBAddresses[BB] = HotDot;
- HotDot += BC.computeCodeSize(BB->begin(), BB->end());
+ const FunctionFragment &MainFragment = Func.getLayout().getMainFragment();
+ const FunctionFragment *ColdFragmentPtr = nullptr;
+ const FunctionFragment *WarmFragmentPtr = nullptr;
+
+ auto tentativeFragmentLayout = [&](const FunctionFragment &Fragment,
+ uint64_t DotAddress) {
+ for (const BinaryBasicBlock *BB : Fragment) {
+ BBAddresses[BB] = DotAddress;
+ DotAddress += BC.computeCodeSize(BB->begin(), BB->end());
}
+ };
+
+ if (!MainFragment.empty()) {
+ assert(HotAddresses.find(&Func) != HotAddresses.end() &&
+ "HotAddresses Should contain Func");
+ uint64_t HotDot = HotAddresses[&Func];
+ tentativeFragmentLayout(MainFragment, HotDot);
+ }
+
+ if ((WarmFragmentPtr = getWarmFragmentOrNullptr(Func.getLayout())) &&
+ !WarmFragmentPtr->empty()) {
+ assert(WarmAddresses.find(&Func) != WarmAddresses.end() &&
+ "WarmAddresses Should contain Func");
+ uint64_t WarmDot = WarmAddresses[&Func];
+ tentativeFragmentLayout(*WarmFragmentPtr, WarmDot);
+ }
+
+ if ((ColdFragmentPtr = getColdFragmentOrNullptr(Func.getLayout())) &&
+ !ColdFragmentPtr->empty()) {
+ assert(ColdAddresses.find(&Func) != ColdAddresses.end() &&
+ "ColdAddresses Should contain Func");
+ uint64_t ColdDot = ColdAddresses[&Func];
+ tentativeFragmentLayout(*ColdFragmentPtr, ColdDot);
}
}
@@ -357,21 +515,60 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
// Hot
CurrentIndex = 0;
- bool ColdLayoutDone = false;
- auto runColdLayout = [&]() {
- DotAddress = tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);
- ColdLayoutDone = true;
- if (opts::HotFunctionsAtEnd)
- DotAddress = alignTo(DotAddress, opts::AlignText);
+ bool ColdWarmLayoutDone = false;
+
+ bool HasWarmFragments = llvm::any_of(SortedFunctions, [&](BinaryFunction *F) {
+ return hasNonEmptyWarmFragment(F);
+ });
+
+ bool HasColdFragments = llvm::any_of(SortedFunctions, [&](BinaryFunction *F) {
+ return hasNonEmptyColdFragment(F);
+ });
+
+ auto runColdWarmLayout = [&]() {
+ if (opts::HotFunctionsAtEnd) {
+ // cold
+ // warm
+ // hot
+ if (HasColdFragments) {
+ DotAddress =
+ tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);
+ DotAddress = alignTo(DotAddress, opts::AlignText);
+ }
+
+ if (HasWarmFragments) {
+ DotAddress =
+ tentativeLayoutRelocWarmPart(BC, SortedFunctions, DotAddress);
+ DotAddress = alignTo(DotAddress, opts::AlignText);
+ }
+ } else {
+ // hot
+ // warm
+ // cold
+ if (HasWarmFragments) {
+ DotAddress = alignTo(DotAddress, opts::AlignText);
+ DotAddress =
+ tentativeLayoutRelocWarmPart(BC, SortedFunctions, DotAddress);
+ }
+
+ if (HasColdFragments) {
+ DotAddress = alignTo(DotAddress, opts::AlignText);
+ DotAddress =
+ tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);
+ }
+ }
+ ColdWarmLayoutDone = true;
};
+
for (BinaryFunction *Func : SortedFunctions) {
if (!BC.shouldEmit(*Func)) {
HotAddresses[Func] = Func->getAddress();
continue;
}
- if (!ColdLayoutDone && CurrentIndex >= LastHotIndex)
- runColdLayout();
+ if (!ColdWarmLayoutDone && CurrentIndex >= LastHotIndex) {
+ runColdWarmLayout();
+ }
DotAddress = alignTo(DotAddress, Func->getMinAlignment());
uint64_t Pad =
@@ -382,18 +579,17 @@ uint64_t LongJmpPass::tentativeLayoutRelocMode(
LLVM_DEBUG(dbgs() << Func->getPrintName() << " tentative: "
<< Twine::utohexstr(DotAddress) << "\n");
if (!Func->isSplit())
- DotAddress += Func->estimateSize();
+ DotAddress += estimateSize(Func);
else
- DotAddress += Func->estimateHotSize();
+ DotAddress += estimateHotSize(Func);
DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());
DotAddress += Func->estimateConstantIslandSize();
++CurrentIndex;
}
-
// Ensure that tentative code layout always runs for cold blocks.
- if (!ColdLayoutDone)
- runColdLayout();
+ if (!ColdWarmLayoutDone)
+ runColdWarmLayout();
// BBs
for (BinaryFunction *Func : SortedFunctions)
@@ -561,11 +757,6 @@ Error LongJmpPass::relax(BinaryFunction &Func, bool &Modified) {
std::vector<std::pair<BinaryBasicBlock *, std::unique_ptr<BinaryBasicBlock>>>
Insertions;
- BinaryBasicBlock *Frontier = getBBAtHotColdSplitPoint(Func);
- uint64_t FrontierAddress = Frontier ? BBAddresses[Frontier] : 0;
- if (FrontierAddress)
- FrontierAddress += Frontier->getNumNonPseudos() * InsnSize;
-
// Add necessary stubs for branch targets we know we can't fit in the
// instruction
for (BinaryBasicBlock &BB : Func) {
@@ -573,6 +764,10 @@ Error LongJmpPass::relax(BinaryFunction &Func, bool &Modified) {
// Stubs themselves are relaxed on the next loop
if (Stubs[&Func].count(&BB))
continue;
+ BinaryBasicBlock *Frontier = getFrontier(Func, BB);
+ uint64_t FrontierAddress = Frontier ? BBAddresses[Frontier] : 0;
+ if (FrontierAddress)
+ FrontierAddress += Frontier->getNumNonPseudos() * InsnSize;
for (MCInst &Inst : BB) {
if (BC.MIB->isPseudo(Inst))
@@ -939,8 +1134,10 @@ Error LongJmpPass::runOnFunctions(BinaryContext &BC) {
} while (Modified);
BC.outs() << "BOLT-INFO: Inserted " << NumHotStubs
<< " stubs in the hot area and " << NumColdStubs
- << " stubs in the cold area. Shared " << NumSharedStubs
- << " times, iterated " << Iterations << " times.\n";
+ << " stubs in the cold area and " << NumWarmStubs
+ << " stubs in the warm area. "
+ << "Shared " << NumSharedStubs << " times, iterated " << Iterations
+ << " times.\n";
return Error::success();
}
} // namespace bolt
diff --git a/bolt/test/AArch64/cdsplit-call-scale.s b/bolt/test/AArch64/cdsplit-call-scale.s
new file mode 100644
index 00000000000000..57fd133823abc6
--- /dev/null
+++ b/bolt/test/AArch64/cdsplit-call-scale.s
@@ -0,0 +1,127 @@
+## Test cdsplit is handled correctly in AArch64
+
+# RUN: llvm-mc --filetype=obj --triple aarch64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.fdata
+# RUN: llvm-strip --strip-unneeded %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --split-functions --split-strategy=cdsplit \
+# RUN: --call-scale=0.0 --print-split --print-only=chain \
+# RUN: --data=%t.fdata --reorder-blocks=ext-tsp \
+# RUN: 2>&1 | FileCheck --check-prefix=LOWINCENTIVE %s
+# RUN: llvm-bolt %t.exe -o %t.bolt --split-functions --split-strategy=cdsplit \
+# RUN: --call-scale=1.0 --print-split --print-only=chain \
+# RUN: --data=%t.fdata --reorder-blocks=ext-tsp \
+# RUN: 2>&1 | FileCheck --check-prefix=MEDINCENTIVE %s
+# RUN: llvm-bolt %t.exe -o %t.bolt --split-functions --split-strategy=cdsplit \
+# RUN: --call-scale=1000.0 --print-split --print-only=chain \
+# RUN: --data=%t.fdata --reorder-blocks=ext-tsp \
+# RUN: 2>&1 | FileCheck --check-prefix=HIGHINCENTIVE %s
+
+# LOWINCENTIVE: Binary Function "chain" after split-functions
+# LOWINCENTIVE: {{^\.Ltmp5}}
+# LOWINCENTIVE: ------- HOT-COLD SPLIT POINT -------
+# LOWINCENTIVE: {{^\.LFT1}}
+
+# MEDINCENTIVE: Binary Function "chain" after split-functions
+# MEDINCENTIVE: {{^\.Ltmp1}}
+# MEDINCENTIVE: ------- HOT-COLD SPLIT POINT -------
+# MEDINCENTIVE: {{^\.LFT1}}
+# MEDINCENTIVE: ------- HOT-COLD SPLIT POINT -------
+# MEDINCENTIVE: {{^\.Ltmp0}}
+# MEDINCENTIVE: {{^\.Ltmp2}}
+# MEDINCENTIVE: {{^\.Ltmp3}}
+# MEDINCENTIVE: {{^\.Ltmp4}}
+# MEDINCENTIVE: {{^\.Ltmp5}}
+
+# HIGHINCENTIVE: Binary Function "chain" after split-functions
+# HIGHINCENTIVE: {{^\.Ltmp1}}
+# HIGHINCENTIVE: ------- HOT-COLD SPLIT POINT -------
+# HIGHINCENTIVE: {{^\.LFT1}}
+# HIGHINCENTIVE: ------- HOT-COLD SPLIT POINT -------
+# HIGHINCENTIVE: {{^\.Ltmp0}}
+# HIGHINCENTIVE: {{^\.Ltmp2}}
+# HIGHINCENTIVE: {{^\.Ltmp3}}
+# HIGHINCENTIVE: {{^\.Ltmp4}}
+# HIGHINCENTIVE: {{^\.Ltmp5}}
+
+ .section .text
+ .globl chain
+ .type chain, %function
+chain:
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+ cmp w0, #2
+LLentry_LLchain_start:
+ b.ge LLchain_start
+# FDATA: 1 chain #LLentry_LLchain_start# 1 chain #LLchain_start# 0 10
+# FDATA: 1 chain #LLentry_LLchain_start# 1 chain #LLfast# 0 500
+LLfast:
+ mov w0, #5
+LLfast_LLexit:
+ b LLexit
+# FDATA: 1 chain #LLfast_LLexit# 1 chain #LLexit# 0 500
+LLchain_start:
+ mov w0, #10
+LLchain_start_LLchain1:
+ b.ge LLchain1
+# FDATA: 1 chain #LLchain_start_LLchain1# 1 chain #LLchain1# 0 10
+# FDATA: 1 chain #LLchain_start_LLchain1# 1 chain #LLcold# 0 0
+LLcold:
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+LLchain1:
+ add w0, w0, #1
+LLchain1_LLchain2:
+ b LLchain2
+# FDATA: 1 chain #LLchain1_LLchain2# 1 chain #LLchain2# 0 10
+LLchain2:
+ add w0, w0, #1
+LLchain2_LLchain3:
+ b LLchain3
+# FDATA: 1 chain #LLchain2_LLchain3# 1 chain #LLchain3# 0 10
+LLchain3:
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+LLchain3_LLchain4:
+ b LLchain4
+# FDATA: 1 chain #LLchain3_LLchain4# 1 chain #LLchain4# 0 10
+LLchain4:
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+LLchain4_LLexit:
+ b LLexit
+# FDATA: 1 chain #LLchain4_LLexit# 1 chain #LLexit# 0 10
+LLexit:
+ ldp x29, x30, [sp], #16
+ ret
+LLchain_end:
+ .size chain, LLchain_end-chain
+
+ .globl main
+ .type main, %function
+main:
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+ mov w0, #1
+LLmain_chain1:
+ bl chain
+# FDATA: 1 main #LLmain_chain1# 1 chain 0 0 500
+ mov w0, #4
+LLmain_chain2:
+ bl chain
+# FDATA: 1 main #LLmain_chain2# 1 chain 0 0 10
+ mov w0, #0
+ ldp x29, x30, [sp], #16
+ ret
+.Lmain_end:
+ .size main, .Lmain_end-main
\ No newline at end of file
diff --git a/bolt/test/AArch64/cdsplit-symbol-names.s b/bolt/test/AArch64/cdsplit-symbol-names.s
new file mode 100644
index 00000000000000..ac2911117b7f24
--- /dev/null
+++ b/bolt/test/AArch64/cdsplit-symbol-names.s
@@ -0,0 +1,98 @@
+## Test the correctness of section names and function symbol names post cdsplit.
+
+
+# RUN: llvm-mc --filetype=obj --triple aarch64-unknown-unknown %s -o %t.o
+# RUN: link_fdata %s %t.o %t.fdata
+# RUN: llvm-strip --strip-unneeded %t.o
+# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt --split-functions --split-strategy=cdsplit \
+# RUN: --data=%t.fdata --reorder-blocks=ext-tsp
+# RUN: llvm-objdump --syms %t.bolt | FileCheck %s --check-prefix=CHECK-SYMS-WARM
+
+# CHECK-SYMS-WARM: 0000000000000000 l df *ABS* 0000000000000000 bolt-pseudo.o
+# CHECK-SYMS-WARM: .text.cold
+# CHECK-SYMS-WARM-SAME: chain.cold
+# CHECK-SYMS-WARM: .text.warm
+# CHECK-SYMS-WARM-SAME: chain.warm
+
+ .section .text
+ .globl chain
+ .type chain, %function
+chain:
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+ cmp w0, #2
+LLentry_LLchain_start:
+ b.ge LLchain_start
+# FDATA: 1 chain #LLentry_LLchain_start# 1 chain #LLchain_start# 0 10
+# FDATA: 1 chain #LLentry_LLchain_start# 1 chain #LLfast# 0 500
+LLfast:
+ mov w0, #5
+LLfast_LLexit:
+ b LLexit
+# FDATA: 1 chain #LLfast_LLexit# 1 chain #LLexit# 0 500
+LLchain_start:
+ mov w0, #10
+LLchain_start_LLchain1:
+ b.ge LLchain1
+# FDATA: 1 chain #LLchain_start_LLchain1# 1 chain #LLchain1# 0 10
+# FDATA: 1 chain #LLchain_start_LLchain1# 1 chain #LLcold# 0 0
+LLcold:
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+LLchain1:
+ add w0, w0, #1
+LLchain1_LLchain2:
+ b LLchain2
+# FDATA: 1 chain #LLchain1_LLchain2# 1 chain #LLchain2# 0 10
+LLchain2:
+ add w0, w0, #1
+LLchain2_LLchain3:
+ b LLchain3
+# FDATA: 1 chain #LLchain2_LLchain3# 1 chain #LLchain3# 0 10
+LLchain3:
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+LLchain3_LLchain4:
+ b LLchain4
+# FDATA: 1 chain #LLchain3_LLchain4# 1 chain #LLchain4# 0 10
+LLchain4:
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+ add w0, w0, #1
+LLchain4_LLexit:
+ b LLexit
+# FDATA: 1 chain #LLchain4_LLexit# 1 chain #LLexit# 0 10
+LLexit:
+ ldp x29, x30, [sp], #16
+ ret
+LLchain_end:
+ .size chain, LLchain_end-chain
+
+ .globl main
+ .type main, %function
+main:
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+ mov w0, #1
+LLmain_chain1:
+ bl chain
+# FDATA: 1 main #LLmain_chain1# 1 chain 0 0 500
+ mov w0, #4
+LLmain_chain2:
+ bl chain
+# FDATA: 1 main #LLmain_chain2# 1 chain 0 0 10
+ mov w0, #0
+ ldp x29, x30, [sp], #16
+ ret
+.Lmain_end:
+ .size main, .Lmain_end-main
\ No newline at end of file
More information about the llvm-commits
mailing list