[llvm-branch-commits] [llvm] [BOLT] Match functions with call graph (PR #98125)
Shaw Young via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Jul 18 10:37:03 PDT 2024
https://github.com/shawbyoung updated https://github.com/llvm/llvm-project/pull/98125
>From cf32a43e7c2b04079c6123fe13df4fb7226d771f Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Tue, 9 Jul 2024 10:04:25 -0700
Subject: [PATCH 01/15] Comments
Created using spr 1.3.4
---
bolt/lib/Profile/YAMLProfileReader.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 69ea0899c5f2c..6753337c24ea7 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -501,7 +501,6 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
// Maps binary functions to adjacent functions in the FCG.
for (const BinaryFunction *CallerBF : BFs) {
- // Add all call targets to the hash map.
for (const BinaryBasicBlock &BB : CallerBF->blocks()) {
for (const MCInst &Inst : BB) {
if (!BC.MIB->isCall(Instr))
@@ -533,7 +532,8 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
}
}
- // Create mapping from neighbor hash to BFs.
+ // Using the constructed adjacent function mapping, creates mapping from
+ // neighbor hash to BFs.
std::unordered_map<uint64_t, std::vector<const BinaryFunction *>>
NeighborHashToBFs;
for (const BinaryFunction *BF : BFs) {
@@ -552,12 +552,12 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
.push_back(BF);
}
- // TODO: change call anchor PR to have this representation - we need it here
+ // TODO: note, this will be introduced in the matching functions with calls
+ // as anchors pr
DenseMap<uint32_t, const yaml::bolt::BinaryFunctionProfile * YamlBF>
IdToYAMLBF;
- // TODO: change call anchor PR to have this representation - we need it here
- // Maps hashes to profiled functions.
+ // Maps YAML functions to adjacent functions in the profile FCG.
std::unordered_map<const yaml::bolt::BinaryFunctionProfile * YamlBF,
FunctionHashes>
YamlBFToHashes(BFs.size());
@@ -590,7 +590,7 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
}
}
- // Matching YAMLBF with neighbor hashes.
+ // Matches YAMLBF to BFs with neighbor hashes.
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (YamlBF.Used)
continue;
>From ee9049fc4bd3d4203c19c9c0982a78ab3b47666f Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Tue, 9 Jul 2024 13:52:05 -0700
Subject: [PATCH 02/15] Moved blended hash definition
Created using spr 1.3.4
---
bolt/include/bolt/Profile/YAMLProfileReader.h | 69 ++++++++++-
bolt/lib/Profile/StaleProfileMatching.cpp | 65 -----------
bolt/lib/Profile/YAMLProfileReader.cpp | 110 ++++++++----------
3 files changed, 119 insertions(+), 125 deletions(-)
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index 36e8f8739eee1..e8a34ecad9a08 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -16,6 +16,73 @@
namespace llvm {
namespace bolt {
+/// An object wrapping several components of a basic block hash. The combined
+/// (blended) hash is represented and stored as one uint64_t, while individual
+/// components are of smaller size (e.g., uint16_t or uint8_t).
+struct BlendedBlockHash {
+private:
+ using ValueOffset = Bitfield::Element<uint16_t, 0, 16>;
+ using ValueOpcode = Bitfield::Element<uint16_t, 16, 16>;
+ using ValueInstr = Bitfield::Element<uint16_t, 32, 16>;
+ using ValuePred = Bitfield::Element<uint8_t, 48, 8>;
+ using ValueSucc = Bitfield::Element<uint8_t, 56, 8>;
+
+public:
+ explicit BlendedBlockHash() {}
+
+ explicit BlendedBlockHash(uint64_t Hash) {
+ Offset = Bitfield::get<ValueOffset>(Hash);
+ OpcodeHash = Bitfield::get<ValueOpcode>(Hash);
+ InstrHash = Bitfield::get<ValueInstr>(Hash);
+ PredHash = Bitfield::get<ValuePred>(Hash);
+ SuccHash = Bitfield::get<ValueSucc>(Hash);
+ }
+
+ /// Combine the blended hash into uint64_t.
+ uint64_t combine() const {
+ uint64_t Hash = 0;
+ Bitfield::set<ValueOffset>(Hash, Offset);
+ Bitfield::set<ValueOpcode>(Hash, OpcodeHash);
+ Bitfield::set<ValueInstr>(Hash, InstrHash);
+ Bitfield::set<ValuePred>(Hash, PredHash);
+ Bitfield::set<ValueSucc>(Hash, SuccHash);
+ return Hash;
+ }
+
+ /// Compute a distance between two given blended hashes. The smaller the
+ /// distance, the more similar two blocks are. For identical basic blocks,
+ /// the distance is zero.
+ uint64_t distance(const BlendedBlockHash &BBH) const {
+ assert(OpcodeHash == BBH.OpcodeHash &&
+ "incorrect blended hash distance computation");
+ uint64_t Dist = 0;
+ // Account for NeighborHash
+ Dist += SuccHash == BBH.SuccHash ? 0 : 1;
+ Dist += PredHash == BBH.PredHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for InstrHash
+ Dist += InstrHash == BBH.InstrHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for Offset
+ Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset);
+ return Dist;
+ }
+
+ /// The offset of the basic block from the function start.
+ uint16_t Offset{0};
+ /// (Loose) Hash of the basic block instructions, excluding operands.
+ uint16_t OpcodeHash{0};
+ /// (Strong) Hash of the basic block instructions, including opcodes and
+ /// operands.
+ uint16_t InstrHash{0};
+ /// (Loose) Hashes of the predecessors of the basic block.
+ uint8_t PredHash{0};
+ /// (Loose) Hashes of the successors of the basic block.
+ uint8_t SuccHash{0};
+};
+
+struct CallGraphMatcher {};
+
class YAMLProfileReader : public ProfileReaderBase {
public:
explicit YAMLProfileReader(StringRef Filename)
@@ -92,7 +159,7 @@ class YAMLProfileReader : public ProfileReaderBase {
/// Matches functions using exact hash.
size_t matchWithHash(BinaryContext &BC);
-
+
/// Matches functions using the call graph.
size_t matchWithCallGraph(BinaryContext &BC);
diff --git a/bolt/lib/Profile/StaleProfileMatching.cpp b/bolt/lib/Profile/StaleProfileMatching.cpp
index e473beb2fad8c..9631ac176554e 100644
--- a/bolt/lib/Profile/StaleProfileMatching.cpp
+++ b/bolt/lib/Profile/StaleProfileMatching.cpp
@@ -121,71 +121,6 @@ cl::opt<unsigned> StaleMatchingCostJumpUnknownFTInc(
namespace llvm {
namespace bolt {
-/// An object wrapping several components of a basic block hash. The combined
-/// (blended) hash is represented and stored as one uint64_t, while individual
-/// components are of smaller size (e.g., uint16_t or uint8_t).
-struct BlendedBlockHash {
-private:
- using ValueOffset = Bitfield::Element<uint16_t, 0, 16>;
- using ValueOpcode = Bitfield::Element<uint16_t, 16, 16>;
- using ValueInstr = Bitfield::Element<uint16_t, 32, 16>;
- using ValuePred = Bitfield::Element<uint8_t, 48, 8>;
- using ValueSucc = Bitfield::Element<uint8_t, 56, 8>;
-
-public:
- explicit BlendedBlockHash() {}
-
- explicit BlendedBlockHash(uint64_t Hash) {
- Offset = Bitfield::get<ValueOffset>(Hash);
- OpcodeHash = Bitfield::get<ValueOpcode>(Hash);
- InstrHash = Bitfield::get<ValueInstr>(Hash);
- PredHash = Bitfield::get<ValuePred>(Hash);
- SuccHash = Bitfield::get<ValueSucc>(Hash);
- }
-
- /// Combine the blended hash into uint64_t.
- uint64_t combine() const {
- uint64_t Hash = 0;
- Bitfield::set<ValueOffset>(Hash, Offset);
- Bitfield::set<ValueOpcode>(Hash, OpcodeHash);
- Bitfield::set<ValueInstr>(Hash, InstrHash);
- Bitfield::set<ValuePred>(Hash, PredHash);
- Bitfield::set<ValueSucc>(Hash, SuccHash);
- return Hash;
- }
-
- /// Compute a distance between two given blended hashes. The smaller the
- /// distance, the more similar two blocks are. For identical basic blocks,
- /// the distance is zero.
- uint64_t distance(const BlendedBlockHash &BBH) const {
- assert(OpcodeHash == BBH.OpcodeHash &&
- "incorrect blended hash distance computation");
- uint64_t Dist = 0;
- // Account for NeighborHash
- Dist += SuccHash == BBH.SuccHash ? 0 : 1;
- Dist += PredHash == BBH.PredHash ? 0 : 1;
- Dist <<= 16;
- // Account for InstrHash
- Dist += InstrHash == BBH.InstrHash ? 0 : 1;
- Dist <<= 16;
- // Account for Offset
- Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset);
- return Dist;
- }
-
- /// The offset of the basic block from the function start.
- uint16_t Offset{0};
- /// (Loose) Hash of the basic block instructions, excluding operands.
- uint16_t OpcodeHash{0};
- /// (Strong) Hash of the basic block instructions, including opcodes and
- /// operands.
- uint16_t InstrHash{0};
- /// (Loose) Hashes of the predecessors of the basic block.
- uint8_t PredHash{0};
- /// (Loose) Hashes of the successors of the basic block.
- uint8_t SuccHash{0};
-};
-
/// The object is used to identify and match basic blocks in a BinaryFunction
/// given their hashes computed on a binary built from several revisions behind
/// release.
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 6753337c24ea7..2cab5c385c3c9 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -458,58 +458,53 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
struct FunctionHashes {
uint64_t Hash{0};
uint64_t AdjacentFunctionHash{0};
- std::set<uint64_t> AdjacentFunctionHashesSet;
+ std::vector<uint64_t> AdjacentFunctionHashesSet;
};
size_t MatchedWithCallGraph = 0;
std::vector<BinaryFunction *> BFs = BC.getAllBinaryFunctions();
- std::unordered_map<const BinaryFunction *, FunctionHashes> BFToHashes(
- BFs.size());
+ std::unordered_map<BinaryFunction *, FunctionHashes> BFToHashes(BFs.size());
// Computes the loose hash, as in the opcode hash, of a binary function.
auto ComputeBFLooseHash = [&](const BinaryFunction *BF) {
- std::string FunctionHashStr = std::accumulate(
- BF->begin(), BF->end(), std::string(""),
- [&](std::string Accum, const BinaryBasicBlock &BB) {
- std::string BlockHashStr = hashBlockLoose(BC, BB);
- uint16_t OpcodeHash;
- if (YamlBP.Header.HashFunction == HashFunction::StdHash)
- OpcodeHash =
- (uint16_t)hash_value(std::hash<std::string>{}(BlockHashStr));
- else if (YamlBP.Header.HashFunction == HashFunction::XXH3)
- OpcodeHash = (uint16_t)llvm::xxh3_64bits(BlockHashStr);
- else
- llvm_unreachable("Unsupported hash function");
- return std::move(Accum) + std::to_string(OpcodeHash);
- });
-
+ std::string FunctionHashStr;
+ for (const BinaryBasicBlock &BB : *BF) {
+ std::string BlockHashStr = hashBlockLoose(BC, BB);
+ uint16_t OpcodeHash;
+ if (YamlBP.Header.HashFunction == HashFunction::StdHash)
+ OpcodeHash =
+ (uint16_t)hash_value(std::hash<std::string>{}(BlockHashStr));
+ else if (YamlBP.Header.HashFunction == HashFunction::XXH3)
+ OpcodeHash = (uint16_t)llvm::xxh3_64bits(BlockHashStr);
+ else
+ llvm_unreachable("Unsupported hash function");
+ FunctionHashStr.append(std::to_string(OpcodeHash));
+ }
return std::hash<std::string>{}(FunctionHashStr);
};
// Computes the loose hash of a function profile.
auto ComputeYamlBFLooseHash =
[&](const yaml::bolt::BinaryFunctionProfile &YamlBF) {
- std::string HashStr = std::to_string(
- YamlBF.Blocks.begin(), YamlBF.Blocks.end(), std::string(""),
- [&](std::string Accum,
- const yaml::bolt::BinaryBasicBlockProfile &YamlBB) {
- BlendedBlockHash YamlHash(YamlBB.Hash);
- return std::move(Accum) + std::to_string(YamlHash.OpcodeHash);
- });
-
+ std::string HashStr;
+ for (auto &YamlBB : YamlBF.Blocks) {
+ BlendedBlockHash YamlHash(YamlBB.Hash);
+ HashStr.append(std::to_string(YamlHash.OpcodeHash));
+ }
return std::hash<std::string>{}(HashStr);
};
// Maps binary functions to adjacent functions in the FCG.
- for (const BinaryFunction *CallerBF : BFs) {
+ for (BinaryFunction *CallerBF : BFs) {
for (const BinaryBasicBlock &BB : CallerBF->blocks()) {
- for (const MCInst &Inst : BB) {
+ for (const MCInst &Instr : BB) {
if (!BC.MIB->isCall(Instr))
continue;
const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr);
if (!CallSymbol)
continue;
- BinaryData *BD =
- BC.getBinaryDataByName(CallSymbol->getName()) if (!BD) continue;
+ BinaryData *BD = BC.getBinaryDataByName(CallSymbol->getName());
+ if (!BD)
+ continue;
BinaryFunction *CalleeBF = BC.getFunctionForSymbol(BD->getSymbol());
if (!CalleeBF)
continue;
@@ -524,9 +519,9 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
CallerBFIt->second = FunctionHashes();
CallerBFIt->second.Hash = ComputeBFLooseHash(CallerBF);
}
- CalleeBFIt->second.AdjacentFunctionHashesSet.insert(
+ CalleeBFIt->second.AdjacentFunctionHashesSet.push_back(
CallerBFIt->second.Hash);
- CallerBFIt->second.AdjacentFunctionHashesSet.insert(
+ CallerBFIt->second.AdjacentFunctionHashesSet.push_back(
CalleeBFIt->second.Hash);
}
}
@@ -534,13 +529,13 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
// Using the constructed adjacent function mapping, creates mapping from
// neighbor hash to BFs.
- std::unordered_map<uint64_t, std::vector<const BinaryFunction *>>
- NeighborHashToBFs;
- for (const BinaryFunction *BF : BFs) {
+ std::unordered_map<uint64_t, std::vector<BinaryFunction *>> NeighborHashToBFs;
+ for (BinaryFunction *BF : BFs) {
auto It = BFToHashes.find(BF);
assert(It != BFToHashes.end() && "All BFs should be processed");
FunctionHashes &FunctionHashes = It->second;
-
+ std::sort(FunctionHashes.AdjacentFunctionHashesSet.begin(),
+ FunctionHashes.AdjacentFunctionHashesSet.end());
std::string AdjacentFunctionHashStr =
std::accumulate(FunctionHashes.AdjacentFunctionHashesSet.begin(),
FunctionHashes.AdjacentFunctionHashesSet.end(),
@@ -554,38 +549,34 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
// TODO: note, this will be introduced in the matching functions with calls
// as anchors pr
- DenseMap<uint32_t, const yaml::bolt::BinaryFunctionProfile * YamlBF>
- IdToYAMLBF;
+ DenseMap<uint32_t, const yaml::bolt::BinaryFunctionProfile *> IdToYAMLBF;
// Maps YAML functions to adjacent functions in the profile FCG.
- std::unordered_map<const yaml::bolt::BinaryFunctionProfile * YamlBF,
- FunctionHashes>
+ std::unordered_map<const yaml::bolt::BinaryFunctionProfile *, FunctionHashes>
YamlBFToHashes(BFs.size());
for (const auto &CallerYamlBF : YamlBP.Functions) {
- if (YamlBF.Used)
- continue;
- for (const auto &YamlBB : CallerYamlBF.BasicBlocks) {
- for (const auto &CallSite : YamlBB.Callsites) {
+ for (const auto &YamlBB : CallerYamlBF.Blocks) {
+ for (const auto &CallSite : YamlBB.CallSites) {
auto IdToYAMLBFIt = IdToYAMLBF.find(CallSite.DestId);
if (IdToYAMLBFIt == IdToYAMLBF.end())
continue;
- auto CalleeBFIt = YamlBFToHashes.find(IdToYAMLBFIt->second);
- if (CalleeBFIt == YamlBFToHashes.end()) {
- CalleeBFIt->second = FunctionHashes();
- CalleeBFIt->second.Hash =
+ auto CalleeYamlBFIt = YamlBFToHashes.find(IdToYAMLBFIt->second);
+ if (CalleeYamlBFIt == YamlBFToHashes.end()) {
+ CalleeYamlBFIt->second = FunctionHashes();
+ CalleeYamlBFIt->second.Hash =
ComputeYamlBFLooseHash(*IdToYAMLBFIt->second);
}
- auto CallerBFIt = YamlBFToHashes.find(&CallerYamlBF);
- if (CallerBFIt == YamlBFToHashes.end()) {
- CallerBFIt->second = FunctionHashes();
- CallerBFIt->second.Hash = ComputeYamlBFLooseHash(CallerBF);
+ auto CallerYamlBFIt = YamlBFToHashes.find(&CallerYamlBF);
+ if (CallerYamlBFIt == YamlBFToHashes.end()) {
+ CallerYamlBFIt->second = FunctionHashes();
+ CallerYamlBFIt->second.Hash = ComputeYamlBFLooseHash(CallerYamlBF);
}
- CalleeBFIt->second.AdjacentFunctionHashesSet.insert(
- CallerBFIt->second.Hash);
- CallerBFIt->second.AdjacentFunctionHashesSet.insert(
- CalleeBFIt->second.Hash);
+ CalleeYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
+ CallerYamlBFIt->second.Hash);
+ CallerYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
+ CalleeYamlBFIt->second.Hash);
}
}
}
@@ -599,7 +590,8 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
assert(It != YamlBFToHashes.end() &&
"All unused functions should be processed");
FunctionHashes &FunctionHashes = It->second;
-
+ std::sort(FunctionHashes.AdjacentFunctionHashesSet.begin(),
+ FunctionHashes.AdjacentFunctionHashesSet.end());
std::string AdjacentFunctionHashStr =
std::accumulate(FunctionHashes.AdjacentFunctionHashesSet.begin(),
FunctionHashes.AdjacentFunctionHashesSet.end(),
@@ -614,7 +606,7 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
if (NeighborHashToBFsIt == NeighborHashToBFs.end())
continue;
- for (const BinaryFunction *BF : NeighborHashToBFsIt->second) {
+ for (BinaryFunction *BF : NeighborHashToBFsIt->second) {
if (!ProfiledFunctions.count(BF) && profileMatches(YamlBF, *BF)) {
matchProfileToFunction(YamlBF, *BF);
++MatchedWithCallGraph;
@@ -668,7 +660,7 @@ size_t YAMLProfileReader::matchWithNameSimilarity(BinaryContext &BC) {
// Maps namespaces to BFs excluding binary functions with no equal sized
// profiled functions belonging to the same namespace.
- / for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
+ for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
std::string DemangledName = BF->getDemangledName();
std::string Namespace = DeriveNameSpace(DemangledName);
>From f8e5530698bc4e83fc270b8444aa3aaf51372df6 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Tue, 9 Jul 2024 15:26:36 -0700
Subject: [PATCH 03/15] Added CallGraphMatcher class
Created using spr 1.3.4
---
bolt/include/bolt/Profile/YAMLProfileReader.h | 52 +++-
bolt/lib/Profile/YAMLProfileReader.cpp | 276 +++++++++---------
2 files changed, 193 insertions(+), 135 deletions(-)
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index e8a34ecad9a08..7433c60c9ec89 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -81,7 +81,47 @@ struct BlendedBlockHash {
uint8_t SuccHash{0};
};
-struct CallGraphMatcher {};
+struct CallGraphMatcher {
+public:
+ /// Computes the loose hash, as in the opcode hash, of a binary function.
+ uint64_t computeBFLooseHash(BinaryContext &BC,
+ yaml::bolt::BinaryProfile &YamlBP,
+ BinaryFunction *BF);
+
+ /// Computes the loose hash of a function profile.
+ uint64_t computeYamlBFLooseHash(yaml::bolt::BinaryFunctionProfile &YamlBF);
+
+ /// Adds edges to the call graph given the callsites of the parameter
+ /// function.
+ void addBFCGEdges(BinaryContext &BC, yaml::bolt::BinaryProfile &YamlBP,
+ BinaryFunction *BF);
+
+ /// Using the constructed adjacent function mapping, creates mapping from
+ /// neighbor hash to BFs.
+ void computeBFNeighborHashes(BinaryContext &BC);
+
+ /// Construct profile FCG.
+ void constructYAMLFCG(
+ yaml::bolt::BinaryProfile &YamlBP,
+ DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *> &IdToYAMLBF);
+ // private:
+ ///
+ struct FunctionHashes {
+ uint64_t Hash{0};
+ uint64_t AdjacentFunctionHash{0};
+ std::vector<uint64_t> AdjacentFunctionHashesSet;
+ };
+
+ ///
+ std::unordered_map<BinaryFunction *, FunctionHashes> BFToHashes;
+
+ ///
+ std::unordered_map<uint64_t, std::vector<BinaryFunction *>> NeighborHashToBFs;
+
+ ///
+ std::unordered_map<const yaml::bolt::BinaryFunctionProfile *, FunctionHashes>
+ YamlBFToHashes;
+};
class YAMLProfileReader : public ProfileReaderBase {
public:
@@ -107,6 +147,9 @@ class YAMLProfileReader : public ProfileReaderBase {
/// Check if the file contains YAML.
static bool isYAML(StringRef Filename);
+ using ProfileLookupMap =
+ DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *>;
+
private:
/// Adjustments for basic samples profiles (without LBR).
bool NormalizeByInsnCount{false};
@@ -123,6 +166,10 @@ class YAMLProfileReader : public ProfileReaderBase {
/// is attributed.
FunctionSet ProfiledFunctions;
+ /// Maps profiled function id to function, for function matching with calls as
+ /// anchors.
+ ProfileLookupMap IdToYamLBF;
+
/// For LTO symbol resolution.
/// Map a common LTO prefix to a list of YAML profiles matching the prefix.
StringMap<std::vector<yaml::bolt::BinaryFunctionProfile *>> LTOCommonNameMap;
@@ -136,6 +183,9 @@ class YAMLProfileReader : public ProfileReaderBase {
/// BinaryFunction pointers indexed by YamlBP functions.
std::vector<BinaryFunction *> ProfileBFs;
+ /// Interface for call graph function matching.
+ CallGraphMatcher CGMatcher;
+
/// Populate \p Function profile with the one supplied in YAML format.
bool parseFunctionProfile(BinaryFunction &Function,
const yaml::bolt::BinaryFunctionProfile &YamlBF);
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 2cab5c385c3c9..ef60a14584bff 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -55,6 +55,120 @@ llvm::cl::opt<bool> ProfileUseDFS("profile-use-dfs",
namespace llvm {
namespace bolt {
+void CallGraphMatcher::addBFCGEdges(BinaryContext &BC,
+ yaml::bolt::BinaryProfile &YamlBP,
+ BinaryFunction *BF) {
+ for (const BinaryBasicBlock &BB : BF->blocks()) {
+ for (const MCInst &Instr : BB) {
+ if (!BC.MIB->isCall(Instr))
+ continue;
+ const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr);
+ if (!CallSymbol)
+ continue;
+ BinaryData *BD = BC.getBinaryDataByName(CallSymbol->getName());
+ if (!BD)
+ continue;
+ BinaryFunction *CalleeBF = BC.getFunctionForSymbol(BD->getSymbol());
+ if (!CalleeBF)
+ continue;
+
+ auto CalleeBFIt = BFToHashes.find(CalleeBF);
+ if (CalleeBFIt == BFToHashes.end()) {
+ CalleeBFIt->second = FunctionHashes();
+ CalleeBFIt->second.Hash = computeBFLooseHash(BC, YamlBP, CalleeBF);
+ }
+ auto CallerBFIt = BFToHashes.find(BF);
+ if (CallerBFIt == BFToHashes.end()) {
+ CallerBFIt->second = FunctionHashes();
+ CallerBFIt->second.Hash = computeBFLooseHash(BC, YamlBP, BF);
+ }
+ CalleeBFIt->second.AdjacentFunctionHashesSet.push_back(
+ CallerBFIt->second.Hash);
+ CallerBFIt->second.AdjacentFunctionHashesSet.push_back(
+ CalleeBFIt->second.Hash);
+ }
+ }
+}
+
+uint64_t CallGraphMatcher::computeBFLooseHash(BinaryContext &BC,
+ yaml::bolt::BinaryProfile &YamlBP,
+ BinaryFunction *BF) {
+
+ std::string FunctionHashStr;
+ for (const BinaryBasicBlock &BB : *BF) {
+ std::string BlockHashStr = hashBlockLoose(BC, BB);
+ uint16_t OpcodeHash;
+ if (YamlBP.Header.HashFunction == HashFunction::StdHash)
+ OpcodeHash = (uint16_t)hash_value(std::hash<std::string>{}(BlockHashStr));
+ else if (YamlBP.Header.HashFunction == HashFunction::XXH3)
+ OpcodeHash = (uint16_t)llvm::xxh3_64bits(BlockHashStr);
+ else
+ llvm_unreachable("Unsupported hash function");
+ FunctionHashStr.append(std::to_string(OpcodeHash));
+ }
+ return std::hash<std::string>{}(FunctionHashStr);
+}
+
+uint64_t CallGraphMatcher::computeYamlBFLooseHash(
+ yaml::bolt::BinaryFunctionProfile &YamlBF) {
+ std::string HashStr;
+ for (auto &YamlBB : YamlBF.Blocks) {
+ BlendedBlockHash YamlHash(YamlBB.Hash);
+ HashStr.append(std::to_string(YamlHash.OpcodeHash));
+ }
+ return std::hash<std::string>{}(HashStr);
+}
+
+void CallGraphMatcher::computeBFNeighborHashes(BinaryContext &BC) {
+ for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
+ auto It = BFToHashes.find(BF);
+ assert(It != BFToHashes.end() && "All BFs should be processed");
+ FunctionHashes &FunctionHashes = It->second;
+ std::sort(FunctionHashes.AdjacentFunctionHashesSet.begin(),
+ FunctionHashes.AdjacentFunctionHashesSet.end());
+ std::string AdjacentFunctionHashStr =
+ std::accumulate(FunctionHashes.AdjacentFunctionHashesSet.begin(),
+ FunctionHashes.AdjacentFunctionHashesSet.end(),
+ std::string(""), [&](std::string Accum, uint64_t Hash) {
+ return Accum + std::to_string(Hash);
+ });
+ NeighborHashToBFs[FunctionHashes.AdjacentFunctionHash =
+ std::hash<std::string>{}(AdjacentFunctionHashStr)]
+ .push_back(BF);
+ }
+}
+
+void CallGraphMatcher::constructYAMLFCG(
+ yaml::bolt::BinaryProfile &YamlBP,
+ DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *> &IdToYAMLBF) {
+ for (auto &CallerYamlBF : YamlBP.Functions) {
+ for (auto &YamlBB : CallerYamlBF.Blocks) {
+ for (auto &CallSite : YamlBB.CallSites) {
+ auto IdToYAMLBFIt = IdToYAMLBF.find(CallSite.DestId);
+ if (IdToYAMLBFIt == IdToYAMLBF.end())
+ continue;
+
+ auto CalleeYamlBFIt = YamlBFToHashes.find(IdToYAMLBFIt->second);
+ if (CalleeYamlBFIt == YamlBFToHashes.end()) {
+ CalleeYamlBFIt->second = FunctionHashes();
+ CalleeYamlBFIt->second.Hash =
+ computeYamlBFLooseHash(*IdToYAMLBFIt->second);
+ }
+ auto CallerYamlBFIt = YamlBFToHashes.find(&CallerYamlBF);
+ if (CallerYamlBFIt == YamlBFToHashes.end()) {
+ CallerYamlBFIt->second = FunctionHashes();
+ CallerYamlBFIt->second.Hash = computeYamlBFLooseHash(CallerYamlBF);
+ }
+
+ CalleeYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
+ CallerYamlBFIt->second.Hash);
+ CallerYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
+ CalleeYamlBFIt->second.Hash);
+ }
+ }
+ }
+}
+
bool YAMLProfileReader::isYAML(const StringRef Filename) {
if (auto MB = MemoryBuffer::getFileOrSTDIN(Filename)) {
StringRef Buffer = (*MB)->getBuffer();
@@ -455,141 +569,20 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
if (!opts::MatchWithCallGraph)
return 0;
- struct FunctionHashes {
- uint64_t Hash{0};
- uint64_t AdjacentFunctionHash{0};
- std::vector<uint64_t> AdjacentFunctionHashesSet;
- };
size_t MatchedWithCallGraph = 0;
- std::vector<BinaryFunction *> BFs = BC.getAllBinaryFunctions();
- std::unordered_map<BinaryFunction *, FunctionHashes> BFToHashes(BFs.size());
-
- // Computes the loose hash, as in the opcode hash, of a binary function.
- auto ComputeBFLooseHash = [&](const BinaryFunction *BF) {
- std::string FunctionHashStr;
- for (const BinaryBasicBlock &BB : *BF) {
- std::string BlockHashStr = hashBlockLoose(BC, BB);
- uint16_t OpcodeHash;
- if (YamlBP.Header.HashFunction == HashFunction::StdHash)
- OpcodeHash =
- (uint16_t)hash_value(std::hash<std::string>{}(BlockHashStr));
- else if (YamlBP.Header.HashFunction == HashFunction::XXH3)
- OpcodeHash = (uint16_t)llvm::xxh3_64bits(BlockHashStr);
- else
- llvm_unreachable("Unsupported hash function");
- FunctionHashStr.append(std::to_string(OpcodeHash));
- }
- return std::hash<std::string>{}(FunctionHashStr);
- };
- // Computes the loose hash of a function profile.
- auto ComputeYamlBFLooseHash =
- [&](const yaml::bolt::BinaryFunctionProfile &YamlBF) {
- std::string HashStr;
- for (auto &YamlBB : YamlBF.Blocks) {
- BlendedBlockHash YamlHash(YamlBB.Hash);
- HashStr.append(std::to_string(YamlHash.OpcodeHash));
- }
- return std::hash<std::string>{}(HashStr);
- };
-
- // Maps binary functions to adjacent functions in the FCG.
- for (BinaryFunction *CallerBF : BFs) {
- for (const BinaryBasicBlock &BB : CallerBF->blocks()) {
- for (const MCInst &Instr : BB) {
- if (!BC.MIB->isCall(Instr))
- continue;
- const MCSymbol *CallSymbol = BC.MIB->getTargetSymbol(Instr);
- if (!CallSymbol)
- continue;
- BinaryData *BD = BC.getBinaryDataByName(CallSymbol->getName());
- if (!BD)
- continue;
- BinaryFunction *CalleeBF = BC.getFunctionForSymbol(BD->getSymbol());
- if (!CalleeBF)
- continue;
-
- auto CalleeBFIt = BFToHashes.find(CalleeBF);
- if (CalleeBFIt == BFToHashes.end()) {
- CalleeBFIt->second = FunctionHashes();
- CalleeBFIt->second.Hash = ComputeBFLooseHash(CalleeBF);
- }
- auto CallerBFIt = BFToHashes.find(CallerBF);
- if (CallerBFIt == BFToHashes.end()) {
- CallerBFIt->second = FunctionHashes();
- CallerBFIt->second.Hash = ComputeBFLooseHash(CallerBF);
- }
- CalleeBFIt->second.AdjacentFunctionHashesSet.push_back(
- CallerBFIt->second.Hash);
- CallerBFIt->second.AdjacentFunctionHashesSet.push_back(
- CalleeBFIt->second.Hash);
- }
- }
- }
-
- // Using the constructed adjacent function mapping, creates mapping from
- // neighbor hash to BFs.
- std::unordered_map<uint64_t, std::vector<BinaryFunction *>> NeighborHashToBFs;
- for (BinaryFunction *BF : BFs) {
- auto It = BFToHashes.find(BF);
- assert(It != BFToHashes.end() && "All BFs should be processed");
- FunctionHashes &FunctionHashes = It->second;
- std::sort(FunctionHashes.AdjacentFunctionHashesSet.begin(),
- FunctionHashes.AdjacentFunctionHashesSet.end());
- std::string AdjacentFunctionHashStr =
- std::accumulate(FunctionHashes.AdjacentFunctionHashesSet.begin(),
- FunctionHashes.AdjacentFunctionHashesSet.end(),
- std::string(""), [&](std::string Accum, uint64_t Hash) {
- return Accum + std::to_string(Hash);
- });
- NeighborHashToBFs[FunctionHashes.AdjacentFunctionHash =
- std::hash<std::string>{}(AdjacentFunctionHashStr)]
- .push_back(BF);
- }
-
- // TODO: note, this will be introduced in the matching functions with calls
- // as anchors pr
- DenseMap<uint32_t, const yaml::bolt::BinaryFunctionProfile *> IdToYAMLBF;
-
- // Maps YAML functions to adjacent functions in the profile FCG.
- std::unordered_map<const yaml::bolt::BinaryFunctionProfile *, FunctionHashes>
- YamlBFToHashes(BFs.size());
- for (const auto &CallerYamlBF : YamlBP.Functions) {
- for (const auto &YamlBB : CallerYamlBF.Blocks) {
- for (const auto &CallSite : YamlBB.CallSites) {
- auto IdToYAMLBFIt = IdToYAMLBF.find(CallSite.DestId);
- if (IdToYAMLBFIt == IdToYAMLBF.end())
- continue;
-
- auto CalleeYamlBFIt = YamlBFToHashes.find(IdToYAMLBFIt->second);
- if (CalleeYamlBFIt == YamlBFToHashes.end()) {
- CalleeYamlBFIt->second = FunctionHashes();
- CalleeYamlBFIt->second.Hash =
- ComputeYamlBFLooseHash(*IdToYAMLBFIt->second);
- }
- auto CallerYamlBFIt = YamlBFToHashes.find(&CallerYamlBF);
- if (CallerYamlBFIt == YamlBFToHashes.end()) {
- CallerYamlBFIt->second = FunctionHashes();
- CallerYamlBFIt->second.Hash = ComputeYamlBFLooseHash(CallerYamlBF);
- }
-
- CalleeYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
- CallerYamlBFIt->second.Hash);
- CallerYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
- CalleeYamlBFIt->second.Hash);
- }
- }
- }
+ CGMatcher.computeBFNeighborHashes(BC);
+ CGMatcher.constructYAMLFCG(YamlBP, IdToYamLBF);
// Matches YAMLBF to BFs with neighbor hashes.
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (YamlBF.Used)
continue;
- auto It = YamlBFToHashes.find(&YamlBF);
- assert(It != YamlBFToHashes.end() &&
+ auto It = CGMatcher.YamlBFToHashes.find(&YamlBF);
+ assert(It != CGMatcher.YamlBFToHashes.end() &&
"All unused functions should be processed");
- FunctionHashes &FunctionHashes = It->second;
+ CallGraphMatcher::FunctionHashes &FunctionHashes = It->second;
std::sort(FunctionHashes.AdjacentFunctionHashesSet.begin(),
FunctionHashes.AdjacentFunctionHashesSet.end());
std::string AdjacentFunctionHashStr =
@@ -602,8 +595,8 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
std::hash<std::string>{}(AdjacentFunctionHashStr);
auto NeighborHashToBFsIt =
- NeighborHashToBFs.find(FunctionHashes.AdjacentFunctionHash);
- if (NeighborHashToBFsIt == NeighborHashToBFs.end())
+ CGMatcher.NeighborHashToBFs.find(FunctionHashes.AdjacentFunctionHash);
+ if (NeighborHashToBFsIt == CGMatcher.NeighborHashToBFs.end())
continue;
for (BinaryFunction *BF : NeighborHashToBFsIt->second) {
@@ -739,12 +732,27 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
}
YamlProfileToFunction.resize(YamlBP.Functions.size() + 1);
- // Computes hash for binary functions.
- if (opts::MatchProfileWithFunctionHash || opts::MatchWithCallGraph) {
- for (auto &[_, BF] : BC.getBinaryFunctions()) {
- BF.computeHash(YamlBP.Header.IsDFSOrder, YamlBP.Header.HashFunction);
- }
- } else if (!opts::IgnoreHash) {
+ // Map profiled function ids to names.
+ for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions)
+ IdToYamLBF[YamlBF.Id] = &YamlBF;
+
+ std::vector<std::function<void(BinaryFunction *)>> BFPreprocessingLambdas;
+ if (opts::MatchProfileWithFunctionHash) {
+ BFPreprocessingLambdas.push_back([&](BinaryFunction *BF) {
+ BF->computeHash(YamlBP.Header.IsDFSOrder, YamlBP.Header.HashFunction);
+ });
+ }
+ if (opts::MatchWithCallGraph) {
+ BFPreprocessingLambdas.push_back(
+ [&](BinaryFunction *BF) { CGMatcher.addBFCGEdges(BC, YamlBP, BF); });
+ }
+
+ // Preprocesses binary functions.
+ for (BinaryFunction *BF : BC.getAllBinaryFunctions())
+ for (auto Lambda : BFPreprocessingLambdas)
+ Lambda(BF);
+
+ if (!opts::MatchProfileWithFunctionHash && !opts::IgnoreHash) {
for (BinaryFunction *BF : ProfileBFs) {
if (!BF)
continue;
>From a27c3cc2d4f278c7737761531d9f084f9436e5d6 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Wed, 10 Jul 2024 13:32:03 -0700
Subject: [PATCH 04/15] Changed neighbor hash defintion from loose hash to name
concatentation
Created using spr 1.3.4
---
bolt/include/bolt/Profile/YAMLProfileReader.h | 110 +++------------
bolt/lib/Profile/StaleProfileMatching.cpp | 65 +++++++++
bolt/lib/Profile/YAMLProfileReader.cpp | 125 +++++-------------
3 files changed, 113 insertions(+), 187 deletions(-)
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index 7433c60c9ec89..9eaa268251cde 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -16,111 +16,35 @@
namespace llvm {
namespace bolt {
-/// An object wrapping several components of a basic block hash. The combined
-/// (blended) hash is represented and stored as one uint64_t, while individual
-/// components are of smaller size (e.g., uint16_t or uint8_t).
-struct BlendedBlockHash {
-private:
- using ValueOffset = Bitfield::Element<uint16_t, 0, 16>;
- using ValueOpcode = Bitfield::Element<uint16_t, 16, 16>;
- using ValueInstr = Bitfield::Element<uint16_t, 32, 16>;
- using ValuePred = Bitfield::Element<uint8_t, 48, 8>;
- using ValueSucc = Bitfield::Element<uint8_t, 56, 8>;
-
-public:
- explicit BlendedBlockHash() {}
-
- explicit BlendedBlockHash(uint64_t Hash) {
- Offset = Bitfield::get<ValueOffset>(Hash);
- OpcodeHash = Bitfield::get<ValueOpcode>(Hash);
- InstrHash = Bitfield::get<ValueInstr>(Hash);
- PredHash = Bitfield::get<ValuePred>(Hash);
- SuccHash = Bitfield::get<ValueSucc>(Hash);
- }
-
- /// Combine the blended hash into uint64_t.
- uint64_t combine() const {
- uint64_t Hash = 0;
- Bitfield::set<ValueOffset>(Hash, Offset);
- Bitfield::set<ValueOpcode>(Hash, OpcodeHash);
- Bitfield::set<ValueInstr>(Hash, InstrHash);
- Bitfield::set<ValuePred>(Hash, PredHash);
- Bitfield::set<ValueSucc>(Hash, SuccHash);
- return Hash;
- }
-
- /// Compute a distance between two given blended hashes. The smaller the
- /// distance, the more similar two blocks are. For identical basic blocks,
- /// the distance is zero.
- uint64_t distance(const BlendedBlockHash &BBH) const {
- assert(OpcodeHash == BBH.OpcodeHash &&
- "incorrect blended hash distance computation");
- uint64_t Dist = 0;
- // Account for NeighborHash
- Dist += SuccHash == BBH.SuccHash ? 0 : 1;
- Dist += PredHash == BBH.PredHash ? 0 : 1;
- Dist <<= 16;
- // Account for InstrHash
- Dist += InstrHash == BBH.InstrHash ? 0 : 1;
- Dist <<= 16;
- // Account for Offset
- Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset);
- return Dist;
- }
-
- /// The offset of the basic block from the function start.
- uint16_t Offset{0};
- /// (Loose) Hash of the basic block instructions, excluding operands.
- uint16_t OpcodeHash{0};
- /// (Strong) Hash of the basic block instructions, including opcodes and
- /// operands.
- uint16_t InstrHash{0};
- /// (Loose) Hashes of the predecessors of the basic block.
- uint8_t PredHash{0};
- /// (Loose) Hashes of the successors of the basic block.
- uint8_t SuccHash{0};
-};
-
struct CallGraphMatcher {
public:
- /// Computes the loose hash, as in the opcode hash, of a binary function.
- uint64_t computeBFLooseHash(BinaryContext &BC,
- yaml::bolt::BinaryProfile &YamlBP,
- BinaryFunction *BF);
-
- /// Computes the loose hash of a function profile.
- uint64_t computeYamlBFLooseHash(yaml::bolt::BinaryFunctionProfile &YamlBF);
-
- /// Adds edges to the call graph given the callsites of the parameter
- /// function.
+ class YAMLProfileReader;
+ /// Adds edges to the binary function call graph given the callsites of the
+ /// parameter function.
void addBFCGEdges(BinaryContext &BC, yaml::bolt::BinaryProfile &YamlBP,
BinaryFunction *BF);
- /// Using the constructed adjacent function mapping, creates mapping from
- /// neighbor hash to BFs.
+ /// Using the constructed binary function call graph, computes and creates
+ /// mappings from "neighbor hash" (composed of the function names of callee
+ /// and caller functions of a function) to binary functions.
void computeBFNeighborHashes(BinaryContext &BC);
- /// Construct profile FCG.
+ /// Constructs the call graph for profile functions.
void constructYAMLFCG(
yaml::bolt::BinaryProfile &YamlBP,
DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *> &IdToYAMLBF);
- // private:
- ///
- struct FunctionHashes {
- uint64_t Hash{0};
- uint64_t AdjacentFunctionHash{0};
- std::vector<uint64_t> AdjacentFunctionHashesSet;
- };
-
- ///
- std::unordered_map<BinaryFunction *, FunctionHashes> BFToHashes;
-
- ///
+
+ /// Adjacency map for binary functions in the call graph.
+ std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>>
+ BFAdjacencyMap;
+
+ /// Maps neighbor hashes to binary functions.
std::unordered_map<uint64_t, std::vector<BinaryFunction *>> NeighborHashToBFs;
- ///
- std::unordered_map<const yaml::bolt::BinaryFunctionProfile *, FunctionHashes>
- YamlBFToHashes;
+ /// Adjacency map for profile functions in the call graph.
+ std::unordered_map<yaml::bolt::BinaryFunctionProfile *,
+ std::vector<yaml::bolt::BinaryFunctionProfile *>>
+ YamlBFAdjacencyMap;
};
class YAMLProfileReader : public ProfileReaderBase {
diff --git a/bolt/lib/Profile/StaleProfileMatching.cpp b/bolt/lib/Profile/StaleProfileMatching.cpp
index 9631ac176554e..e473beb2fad8c 100644
--- a/bolt/lib/Profile/StaleProfileMatching.cpp
+++ b/bolt/lib/Profile/StaleProfileMatching.cpp
@@ -121,6 +121,71 @@ cl::opt<unsigned> StaleMatchingCostJumpUnknownFTInc(
namespace llvm {
namespace bolt {
+/// An object wrapping several components of a basic block hash. The combined
+/// (blended) hash is represented and stored as one uint64_t, while individual
+/// components are of smaller size (e.g., uint16_t or uint8_t).
+struct BlendedBlockHash {
+private:
+ using ValueOffset = Bitfield::Element<uint16_t, 0, 16>;
+ using ValueOpcode = Bitfield::Element<uint16_t, 16, 16>;
+ using ValueInstr = Bitfield::Element<uint16_t, 32, 16>;
+ using ValuePred = Bitfield::Element<uint8_t, 48, 8>;
+ using ValueSucc = Bitfield::Element<uint8_t, 56, 8>;
+
+public:
+ explicit BlendedBlockHash() {}
+
+ explicit BlendedBlockHash(uint64_t Hash) {
+ Offset = Bitfield::get<ValueOffset>(Hash);
+ OpcodeHash = Bitfield::get<ValueOpcode>(Hash);
+ InstrHash = Bitfield::get<ValueInstr>(Hash);
+ PredHash = Bitfield::get<ValuePred>(Hash);
+ SuccHash = Bitfield::get<ValueSucc>(Hash);
+ }
+
+ /// Combine the blended hash into uint64_t.
+ uint64_t combine() const {
+ uint64_t Hash = 0;
+ Bitfield::set<ValueOffset>(Hash, Offset);
+ Bitfield::set<ValueOpcode>(Hash, OpcodeHash);
+ Bitfield::set<ValueInstr>(Hash, InstrHash);
+ Bitfield::set<ValuePred>(Hash, PredHash);
+ Bitfield::set<ValueSucc>(Hash, SuccHash);
+ return Hash;
+ }
+
+ /// Compute a distance between two given blended hashes. The smaller the
+ /// distance, the more similar two blocks are. For identical basic blocks,
+ /// the distance is zero.
+ uint64_t distance(const BlendedBlockHash &BBH) const {
+ assert(OpcodeHash == BBH.OpcodeHash &&
+ "incorrect blended hash distance computation");
+ uint64_t Dist = 0;
+ // Account for NeighborHash
+ Dist += SuccHash == BBH.SuccHash ? 0 : 1;
+ Dist += PredHash == BBH.PredHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for InstrHash
+ Dist += InstrHash == BBH.InstrHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for Offset
+ Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset);
+ return Dist;
+ }
+
+ /// The offset of the basic block from the function start.
+ uint16_t Offset{0};
+ /// (Loose) Hash of the basic block instructions, excluding operands.
+ uint16_t OpcodeHash{0};
+ /// (Strong) Hash of the basic block instructions, including opcodes and
+ /// operands.
+ uint16_t InstrHash{0};
+ /// (Loose) Hashes of the predecessors of the basic block.
+ uint8_t PredHash{0};
+ /// (Loose) Hashes of the successors of the basic block.
+ uint8_t SuccHash{0};
+};
+
/// The object is used to identify and match basic blocks in a BinaryFunction
/// given their hashes computed on a binary built from several revisions behind
/// release.
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index ef60a14584bff..e5062578e1eca 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -72,69 +72,26 @@ void CallGraphMatcher::addBFCGEdges(BinaryContext &BC,
if (!CalleeBF)
continue;
- auto CalleeBFIt = BFToHashes.find(CalleeBF);
- if (CalleeBFIt == BFToHashes.end()) {
- CalleeBFIt->second = FunctionHashes();
- CalleeBFIt->second.Hash = computeBFLooseHash(BC, YamlBP, CalleeBF);
- }
- auto CallerBFIt = BFToHashes.find(BF);
- if (CallerBFIt == BFToHashes.end()) {
- CallerBFIt->second = FunctionHashes();
- CallerBFIt->second.Hash = computeBFLooseHash(BC, YamlBP, BF);
- }
- CalleeBFIt->second.AdjacentFunctionHashesSet.push_back(
- CallerBFIt->second.Hash);
- CallerBFIt->second.AdjacentFunctionHashesSet.push_back(
- CalleeBFIt->second.Hash);
+ BFAdjacencyMap[CalleeBF].push_back(BF);
+ BFAdjacencyMap[BF].push_back(CalleeBF);
}
}
}
-uint64_t CallGraphMatcher::computeBFLooseHash(BinaryContext &BC,
- yaml::bolt::BinaryProfile &YamlBP,
- BinaryFunction *BF) {
-
- std::string FunctionHashStr;
- for (const BinaryBasicBlock &BB : *BF) {
- std::string BlockHashStr = hashBlockLoose(BC, BB);
- uint16_t OpcodeHash;
- if (YamlBP.Header.HashFunction == HashFunction::StdHash)
- OpcodeHash = (uint16_t)hash_value(std::hash<std::string>{}(BlockHashStr));
- else if (YamlBP.Header.HashFunction == HashFunction::XXH3)
- OpcodeHash = (uint16_t)llvm::xxh3_64bits(BlockHashStr);
- else
- llvm_unreachable("Unsupported hash function");
- FunctionHashStr.append(std::to_string(OpcodeHash));
- }
- return std::hash<std::string>{}(FunctionHashStr);
-}
-
-uint64_t CallGraphMatcher::computeYamlBFLooseHash(
- yaml::bolt::BinaryFunctionProfile &YamlBF) {
- std::string HashStr;
- for (auto &YamlBB : YamlBF.Blocks) {
- BlendedBlockHash YamlHash(YamlBB.Hash);
- HashStr.append(std::to_string(YamlHash.OpcodeHash));
- }
- return std::hash<std::string>{}(HashStr);
-}
-
void CallGraphMatcher::computeBFNeighborHashes(BinaryContext &BC) {
for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
- auto It = BFToHashes.find(BF);
- assert(It != BFToHashes.end() && "All BFs should be processed");
- FunctionHashes &FunctionHashes = It->second;
- std::sort(FunctionHashes.AdjacentFunctionHashesSet.begin(),
- FunctionHashes.AdjacentFunctionHashesSet.end());
- std::string AdjacentFunctionHashStr =
- std::accumulate(FunctionHashes.AdjacentFunctionHashesSet.begin(),
- FunctionHashes.AdjacentFunctionHashesSet.end(),
- std::string(""), [&](std::string Accum, uint64_t Hash) {
- return Accum + std::to_string(Hash);
- });
- NeighborHashToBFs[FunctionHashes.AdjacentFunctionHash =
- std::hash<std::string>{}(AdjacentFunctionHashStr)]
- .push_back(BF);
+ auto It = BFAdjacencyMap.find(BF);
+ assert(It != BFAdjacencyMap.end() && "All BFs should be processed");
+ std::vector<BinaryFunction *> &AdjacentBFs = It->second;
+ std::sort(AdjacentBFs.begin(), AdjacentBFs.end(),
+ [&](const BinaryFunction *A, const BinaryFunction *B) {
+ return A->getOneName() < B->getOneName();
+ });
+ std::string HashStr;
+ for (BinaryFunction *BF : AdjacentBFs)
+ HashStr += BF->getOneName();
+ uint64_t Hash = std::hash<std::string>{}(HashStr);
+ NeighborHashToBFs[Hash].push_back(BF);
}
}
@@ -147,23 +104,7 @@ void CallGraphMatcher::constructYAMLFCG(
auto IdToYAMLBFIt = IdToYAMLBF.find(CallSite.DestId);
if (IdToYAMLBFIt == IdToYAMLBF.end())
continue;
-
- auto CalleeYamlBFIt = YamlBFToHashes.find(IdToYAMLBFIt->second);
- if (CalleeYamlBFIt == YamlBFToHashes.end()) {
- CalleeYamlBFIt->second = FunctionHashes();
- CalleeYamlBFIt->second.Hash =
- computeYamlBFLooseHash(*IdToYAMLBFIt->second);
- }
- auto CallerYamlBFIt = YamlBFToHashes.find(&CallerYamlBF);
- if (CallerYamlBFIt == YamlBFToHashes.end()) {
- CallerYamlBFIt->second = FunctionHashes();
- CallerYamlBFIt->second.Hash = computeYamlBFLooseHash(CallerYamlBF);
- }
-
- CalleeYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
- CallerYamlBFIt->second.Hash);
- CallerYamlBFIt->second.AdjacentFunctionHashesSet.push_back(
- CalleeYamlBFIt->second.Hash);
+ YamlBFAdjacencyMap[&CallerYamlBF].push_back(IdToYAMLBFIt->second);
}
}
}
@@ -569,36 +510,32 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
if (!opts::MatchWithCallGraph)
return 0;
- size_t MatchedWithCallGraph = 0;
-
CGMatcher.computeBFNeighborHashes(BC);
CGMatcher.constructYAMLFCG(YamlBP, IdToYamLBF);
+ size_t MatchedWithCallGraph = 0;
// Matches YAMLBF to BFs with neighbor hashes.
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (YamlBF.Used)
continue;
-
- auto It = CGMatcher.YamlBFToHashes.find(&YamlBF);
- assert(It != CGMatcher.YamlBFToHashes.end() &&
+ auto It = CGMatcher.YamlBFAdjacencyMap.find(&YamlBF);
+ assert(It != CGMatcher.YamlBFAdjacencyMap.end() &&
"All unused functions should be processed");
- CallGraphMatcher::FunctionHashes &FunctionHashes = It->second;
- std::sort(FunctionHashes.AdjacentFunctionHashesSet.begin(),
- FunctionHashes.AdjacentFunctionHashesSet.end());
- std::string AdjacentFunctionHashStr =
- std::accumulate(FunctionHashes.AdjacentFunctionHashesSet.begin(),
- FunctionHashes.AdjacentFunctionHashesSet.end(),
- std::string(""), [&](std::string Accum, uint64_t Hash) {
- return Accum + std::to_string(Hash);
- });
- FunctionHashes.AdjacentFunctionHash =
- std::hash<std::string>{}(AdjacentFunctionHashStr);
-
- auto NeighborHashToBFsIt =
- CGMatcher.NeighborHashToBFs.find(FunctionHashes.AdjacentFunctionHash);
+ std::vector<yaml::bolt::BinaryFunctionProfile *> &AdjacentFunctions =
+ It->second;
+ std::sort(AdjacentFunctions.begin(), AdjacentFunctions.end(),
+ [&](const yaml::bolt::BinaryFunctionProfile *A,
+ const yaml::bolt::BinaryFunctionProfile *B) {
+ return A->Name < B->Name;
+ });
+ std::string AdjacentFunctionHashStr;
+ for (auto &AdjacentFunction : AdjacentFunctions) {
+ AdjacentFunctionHashStr += AdjacentFunction->Name;
+ }
+ uint64_t Hash = std::hash<std::string>{}(AdjacentFunctionHashStr);
+ auto NeighborHashToBFsIt = CGMatcher.NeighborHashToBFs.find(Hash);
if (NeighborHashToBFsIt == CGMatcher.NeighborHashToBFs.end())
continue;
-
for (BinaryFunction *BF : NeighborHashToBFsIt->second) {
if (!ProfiledFunctions.count(BF) && profileMatches(YamlBF, *BF)) {
matchProfileToFunction(YamlBF, *BF);
>From 3552b317a7a2960b3cad54542d2cc2f5ce831dae Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Wed, 10 Jul 2024 13:33:08 -0700
Subject: [PATCH 05/15] Rm unecessary class decl
Created using spr 1.3.4
---
bolt/include/bolt/Profile/YAMLProfileReader.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index 9eaa268251cde..8a8e5ccb77282 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -18,7 +18,6 @@ namespace bolt {
struct CallGraphMatcher {
public:
- class YAMLProfileReader;
/// Adds edges to the binary function call graph given the callsites of the
/// parameter function.
void addBFCGEdges(BinaryContext &BC, yaml::bolt::BinaryProfile &YamlBP,
>From d1b2c830bb76f903a09baa6a144f2f406c50a0f8 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Thu, 11 Jul 2024 09:57:24 -0700
Subject: [PATCH 06/15] Changed adjacent function rep to sets
Created using spr 1.3.4
---
bolt/include/bolt/Profile/YAMLProfileReader.h | 4 +--
bolt/lib/Profile/YAMLProfileReader.cpp | 27 +++++++------------
2 files changed, 12 insertions(+), 19 deletions(-)
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index 8a8e5ccb77282..55113a4b53060 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -34,7 +34,7 @@ struct CallGraphMatcher {
DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *> &IdToYAMLBF);
/// Adjacency map for binary functions in the call graph.
- std::unordered_map<BinaryFunction *, std::vector<BinaryFunction *>>
+ std::unordered_map<BinaryFunction *, std::set<BinaryFunction *>>
BFAdjacencyMap;
/// Maps neighbor hashes to binary functions.
@@ -42,7 +42,7 @@ struct CallGraphMatcher {
/// Adjacency map for profile functions in the call graph.
std::unordered_map<yaml::bolt::BinaryFunctionProfile *,
- std::vector<yaml::bolt::BinaryFunctionProfile *>>
+ std::set<yaml::bolt::BinaryFunctionProfile *>>
YamlBFAdjacencyMap;
};
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index fe2126c21cc21..26b56cf98206d 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -71,8 +71,8 @@ void CallGraphMatcher::addBFCGEdges(BinaryContext &BC,
if (!CalleeBF)
continue;
- BFAdjacencyMap[CalleeBF].push_back(BF);
- BFAdjacencyMap[BF].push_back(CalleeBF);
+ BFAdjacencyMap[CalleeBF].insert(BF);
+ BFAdjacencyMap[BF].insert(CalleeBF);
}
}
}
@@ -80,12 +80,9 @@ void CallGraphMatcher::addBFCGEdges(BinaryContext &BC,
void CallGraphMatcher::computeBFNeighborHashes(BinaryContext &BC) {
for (BinaryFunction *BF : BC.getAllBinaryFunctions()) {
auto It = BFAdjacencyMap.find(BF);
- assert(It != BFAdjacencyMap.end() && "All BFs should be processed");
- std::vector<BinaryFunction *> &AdjacentBFs = It->second;
- std::sort(AdjacentBFs.begin(), AdjacentBFs.end(),
- [&](const BinaryFunction *A, const BinaryFunction *B) {
- return A->getOneName() < B->getOneName();
- });
+ if (It == BFAdjacencyMap.end())
+ continue;
+ auto &AdjacentBFs = It->second;
std::string HashStr;
for (BinaryFunction *BF : AdjacentBFs)
HashStr += BF->getOneName();
@@ -103,7 +100,8 @@ void CallGraphMatcher::constructYAMLFCG(
auto IdToYAMLBFIt = IdToYAMLBF.find(CallSite.DestId);
if (IdToYAMLBFIt == IdToYAMLBF.end())
continue;
- YamlBFAdjacencyMap[&CallerYamlBF].push_back(IdToYAMLBFIt->second);
+ YamlBFAdjacencyMap[&CallerYamlBF].insert(IdToYAMLBFIt->second);
+ YamlBFAdjacencyMap[IdToYAMLBFIt->second].insert(&CallerYamlBF);
}
}
}
@@ -518,15 +516,10 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
if (YamlBF.Used)
continue;
auto It = CGMatcher.YamlBFAdjacencyMap.find(&YamlBF);
- assert(It != CGMatcher.YamlBFAdjacencyMap.end() &&
- "All unused functions should be processed");
- std::vector<yaml::bolt::BinaryFunctionProfile *> &AdjacentFunctions =
+ if (It == CGMatcher.YamlBFAdjacencyMap.end())
+ continue;
+ std::set<yaml::bolt::BinaryFunctionProfile *> &AdjacentFunctions =
It->second;
- std::sort(AdjacentFunctions.begin(), AdjacentFunctions.end(),
- [&](const yaml::bolt::BinaryFunctionProfile *A,
- const yaml::bolt::BinaryFunctionProfile *B) {
- return A->Name < B->Name;
- });
std::string AdjacentFunctionHashStr;
for (auto &AdjacentFunction : AdjacentFunctions) {
AdjacentFunctionHashStr += AdjacentFunction->Name;
>From d83d604d6677ea90fafae8202d014da339502568 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Thu, 11 Jul 2024 10:22:14 -0700
Subject: [PATCH 07/15] Added test
Created using spr 1.3.4
---
bolt/lib/Profile/YAMLProfileReader.cpp | 2 +-
.../X86/match-functions-with-call-graph.test | 104 ++++++++++++++++++
2 files changed, 105 insertions(+), 1 deletion(-)
create mode 100644 bolt/test/X86/match-functions-with-call-graph.test
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 26b56cf98206d..2fa31f9aecbbf 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -529,7 +529,7 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
if (NeighborHashToBFsIt == CGMatcher.NeighborHashToBFs.end())
continue;
for (BinaryFunction *BF : NeighborHashToBFsIt->second) {
- if (!ProfiledFunctions.count(BF) && profileMatches(YamlBF, *BF)) {
+ if (!ProfiledFunctions.count(BF)) {
matchProfileToFunction(YamlBF, *BF);
++MatchedWithCallGraph;
}
diff --git a/bolt/test/X86/match-functions-with-call-graph.test b/bolt/test/X86/match-functions-with-call-graph.test
new file mode 100644
index 0000000000000..6a491f3059a31
--- /dev/null
+++ b/bolt/test/X86/match-functions-with-call-graph.test
@@ -0,0 +1,104 @@
+## Tests blocks matching by called function names in inferStaleProfile.
+
+# REQUIRES: system-linux
+# RUN: split-file %s %t
+# RUN: %clang %cflags %t/main.cpp -o %t.exe -Wl,-q -nostdlib
+# RUN: llvm-bolt %t.exe -o %t.out --data %t/yaml --profile-ignore-hash -v=1 \
+# RUN: --dyno-stats --print-cfg --infer-stale-profile=1 --match-with-call-graph 2>&1 | FileCheck %s
+
+# CHECK: BOLT-INFO: applying profile inference for "qux"
+
+
+#--- main.cpp
+void foo() {}
+
+void bar() {}
+
+void qux() {
+ foo();
+ bar();
+}
+
+void fred() {
+ foo();
+ qux();
+ bar();
+ bar();
+ foo();
+}
+
+int main() {
+ return 0;
+}
+
+#--- yaml
+---
+header:
+ profile-version: 1
+ binary-name: 'match-functions-with-calls-as-anchors.s.tmp.exe'
+ binary-build-id: '<unknown>'
+ profile-flags: [ lbr ]
+ profile-origin: branch profile reader
+ profile-events: ''
+ dfs-order: false
+ hash-func: xxh3
+functions:
+ - name: main
+ fid: 0
+ hash: 0x0000000000000001
+ exec: 1
+ nblocks: 6
+ blocks:
+ - bid: 1
+ hash: 0x0000000000000001
+ insns: 1
+ succ: [ { bid: 3, cnt: 1} ]
+ - name: _Z3foov
+ fid: 1
+ hash: 0x0000000000000002
+ exec: 1
+ nblocks: 6
+ blocks:
+ - bid: 1
+ hash: 0x0000000000000002
+ insns: 1
+ succ: [ { bid: 3, cnt: 1} ]
+
+ - name: _Z3barv
+ fid: 2
+ hash: 0x0000000000000003
+ exec: 1
+ nblocks: 6
+ blocks:
+ - bid: 1
+ hash: 0x0000000000000003
+ insns: 1
+ succ: [ { bid: 3, cnt: 1} ]
+ - name: _Z3quxv
+ fid: 3
+ hash: 0x0000000000000004
+ exec: 4
+ nblocks: 6
+ blocks:
+ - bid: 1
+ hash: 0x0000000000000004
+ insns: 1
+ succ: [ { bid: 3, cnt: 1} ]
+ calls: [ { off : 0, fid : 1, cnt : 0},
+ { off : 0, fid : 2, cnt : 0} ]
+ - name: _Z4fredv
+ fid: 4
+ hash: 0x0000000000000005
+ exec: 1
+ nblocks: 6
+ blocks:
+ - bid: 1
+ hash: 0x0000000000000005
+ insns: 1
+ succ: [ { bid: 3, cnt: 1} ]
+ calls: [ { off : 0, fid : 3, cnt : 0},
+ { off : 0, fid : 1, cnt : 0},
+ { off : 0, fid : 2, cnt : 0},
+ { off : 0, fid : 1, cnt : 0},
+ { off : 0, fid : 2, cnt : 0} ]
+...
>From 294f108f54311077768a8685a74fff64a02ec558 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Thu, 11 Jul 2024 10:29:53 -0700
Subject: [PATCH 08/15] Test fix
Created using spr 1.3.4
---
bolt/test/X86/match-functions-with-call-graph.test | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/bolt/test/X86/match-functions-with-call-graph.test b/bolt/test/X86/match-functions-with-call-graph.test
index 6a491f3059a31..3e37791128d3f 100644
--- a/bolt/test/X86/match-functions-with-call-graph.test
+++ b/bolt/test/X86/match-functions-with-call-graph.test
@@ -6,8 +6,7 @@
# RUN: llvm-bolt %t.exe -o %t.out --data %t/yaml --profile-ignore-hash -v=1 \
# RUN: --dyno-stats --print-cfg --infer-stale-profile=1 --match-with-call-graph 2>&1 | FileCheck %s
-# CHECK: BOLT-INFO: applying profile inference for "qux"
-
+# CHECK: BOLT-INFO: matched 4 functions with call graph
#--- main.cpp
void foo() {}
>From 1f9165f93ba3fef401e8ad9f110b6a84198c1d15 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Thu, 11 Jul 2024 10:46:00 -0700
Subject: [PATCH 09/15] Added block count heuristic for cg matching
Created using spr 1.3.4
---
bolt/lib/Profile/YAMLProfileReader.cpp | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 2fa31f9aecbbf..1c4c3194404e9 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -528,12 +528,24 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
auto NeighborHashToBFsIt = CGMatcher.NeighborHashToBFs.find(Hash);
if (NeighborHashToBFsIt == CGMatcher.NeighborHashToBFs.end())
continue;
+
+ BinaryFunction *ClosestBF = nullptr;
+ size_t MinDistance = std::numeric_limits<size_t>::max();
for (BinaryFunction *BF : NeighborHashToBFsIt->second) {
- if (!ProfiledFunctions.count(BF)) {
- matchProfileToFunction(YamlBF, *BF);
- ++MatchedWithCallGraph;
+ if (ProfiledFunctions.count(BF))
+ continue;
+ size_t Distance = YamlBF.NumBasicBlocks > BF->size()
+ ? YamlBF.NumBasicBlocks - BF->size()
+ : BF->size() - YamlBF.NumBasicBlocks;
+ if (Distance < MinDistance) {
+ MinDistance = Distance;
+ ClosestBF = BF;
}
}
+ if (ClosestBF) {
+ matchProfileToFunction(YamlBF, *ClosestBF);
+ ++MatchedWithCallGraph;
+ }
}
return MatchedWithCallGraph;
>From 92eaec0a918faefbd4984865f8e86a4c36fce5c1 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Thu, 11 Jul 2024 11:42:32 -0700
Subject: [PATCH 10/15] Comments
Created using spr 1.3.4
---
bolt/include/bolt/Profile/YAMLProfileReader.h | 1 +
bolt/lib/Profile/YAMLProfileReader.cpp | 9 +++++++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index 55113a4b53060..468bbbc84f670 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -16,6 +16,7 @@
namespace llvm {
namespace bolt {
+/// A class for matching binary functions in functions in the YAML profile.
struct CallGraphMatcher {
public:
/// Adds edges to the binary function call graph given the callsites of the
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 1c4c3194404e9..90621e03fa498 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -507,10 +507,10 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
if (!opts::MatchWithCallGraph)
return 0;
+ size_t MatchedWithCallGraph = 0;
CGMatcher.computeBFNeighborHashes(BC);
CGMatcher.constructYAMLFCG(YamlBP, IdToYamLBF);
- size_t MatchedWithCallGraph = 0;
// Matches YAMLBF to BFs with neighbor hashes.
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (YamlBF.Used)
@@ -518,6 +518,7 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
auto It = CGMatcher.YamlBFAdjacencyMap.find(&YamlBF);
if (It == CGMatcher.YamlBFAdjacencyMap.end())
continue;
+ // Computes profiled function's neighbor hash.
std::set<yaml::bolt::BinaryFunctionProfile *> &AdjacentFunctions =
It->second;
std::string AdjacentFunctionHashStr;
@@ -528,7 +529,8 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
auto NeighborHashToBFsIt = CGMatcher.NeighborHashToBFs.find(Hash);
if (NeighborHashToBFsIt == CGMatcher.NeighborHashToBFs.end())
continue;
-
+ // Finds the binary function with the closest block size to the profiled
+ // function and matches.
BinaryFunction *ClosestBF = nullptr;
size_t MinDistance = std::numeric_limits<size_t>::max();
for (BinaryFunction *BF : NeighborHashToBFsIt->second) {
@@ -677,6 +679,9 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions)
IdToYamLBF[YamlBF.Id] = &YamlBF;
+ // Creates a vector of lamdas that preprocess binary functions for function
+ // matching to avoid multiple preprocessing passes over binary functions in
+ // different function matching techniques.
std::vector<std::function<void(BinaryFunction *)>> BFPreprocessingLambdas;
if (opts::MatchProfileWithFunctionHash) {
BFPreprocessingLambdas.push_back([&](BinaryFunction *BF) {
>From a6aa61d33e4d5598d152326b056b8ab586cb6d96 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Mon, 15 Jul 2024 07:23:44 -0700
Subject: [PATCH 11/15] Updated cmd line docs, changed unordered_map to
DenseMap in CGMatcher, updated variable names
Created using spr 1.3.4
---
bolt/docs/CommandLineArgumentReference.md | 4 ++++
bolt/include/bolt/Profile/YAMLProfileReader.h | 6 +++---
bolt/lib/Profile/YAMLProfileReader.cpp | 10 +++++-----
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md
index 17c52c65e472f..a7f73d6422c11 100644
--- a/bolt/docs/CommandLineArgumentReference.md
+++ b/bolt/docs/CommandLineArgumentReference.md
@@ -680,6 +680,10 @@
threshold means fewer functions to process. E.g threshold of 90 means only top
10 percent of functions with profile will be processed.
+- `--match-with-call-graph`
+
+ Match functions with call graph
+
- `--memcpy1-spec=<func1,func2:cs1:cs2,func3:cs1,...>`
List of functions with call sites for which to specialize memcpy() for size 1
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index 468bbbc84f670..c5158f2f9ee76 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -35,14 +35,14 @@ struct CallGraphMatcher {
DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *> &IdToYAMLBF);
/// Adjacency map for binary functions in the call graph.
- std::unordered_map<BinaryFunction *, std::set<BinaryFunction *>>
+ DenseMap<BinaryFunction *, std::set<BinaryFunction *>>
BFAdjacencyMap;
/// Maps neighbor hashes to binary functions.
- std::unordered_map<uint64_t, std::vector<BinaryFunction *>> NeighborHashToBFs;
+ DenseMap<uint64_t, std::vector<BinaryFunction *>> NeighborHashToBFs;
/// Adjacency map for profile functions in the call graph.
- std::unordered_map<yaml::bolt::BinaryFunctionProfile *,
+ DenseMap<yaml::bolt::BinaryFunctionProfile *,
std::set<yaml::bolt::BinaryFunctionProfile *>>
YamlBFAdjacencyMap;
};
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 90621e03fa498..565ef26fea898 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -682,21 +682,21 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) {
// Creates a vector of lamdas that preprocess binary functions for function
// matching to avoid multiple preprocessing passes over binary functions in
// different function matching techniques.
- std::vector<std::function<void(BinaryFunction *)>> BFPreprocessingLambdas;
+ std::vector<std::function<void(BinaryFunction *)>> BFPreprocessingFuncs;
if (opts::MatchProfileWithFunctionHash) {
- BFPreprocessingLambdas.push_back([&](BinaryFunction *BF) {
+ BFPreprocessingFuncs.push_back([&](BinaryFunction *BF) {
BF->computeHash(YamlBP.Header.IsDFSOrder, YamlBP.Header.HashFunction);
});
}
if (opts::MatchWithCallGraph) {
- BFPreprocessingLambdas.push_back(
+ BFPreprocessingFuncs.push_back(
[&](BinaryFunction *BF) { CGMatcher.addBFCGEdges(BC, YamlBP, BF); });
}
// Preprocesses binary functions.
for (BinaryFunction *BF : BC.getAllBinaryFunctions())
- for (auto Lambda : BFPreprocessingLambdas)
- Lambda(BF);
+ for (auto BFPreprocessingFunc : BFPreprocessingFuncs)
+ BFPreprocessingFunc(BF);
if (!opts::MatchProfileWithFunctionHash && !opts::IgnoreHash) {
for (BinaryFunction *BF : ProfileBFs) {
>From 9ec7044bd6e5ad29472dc000cc47786ddab7cf04 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Tue, 16 Jul 2024 09:15:51 -0700
Subject: [PATCH 12/15] Removed unnecessary braces
Created using spr 1.3.4
---
bolt/lib/Profile/YAMLProfileReader.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 32d881ece3737..16d9b103d7a14 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -529,9 +529,8 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
if (!AdjacentYamlBFs)
continue;
std::string AdjacentYamlBFsHashStr;
- for (auto *AdjacentYamlBF : *AdjacentYamlBFs) {
+ for (auto *AdjacentYamlBF : *AdjacentYamlBFs)
AdjacentYamlBFsHashStr += AdjacentYamlBF->Name;
- }
uint64_t Hash = std::hash<std::string>{}(AdjacentYamlBFsHashStr);
std::vector<BinaryFunction *> *BFsWithSameHash =
CGMatcher.getBFsWithNeighborHash(Hash);
>From 8804dcb373d6cc68a4ee4cf2b4662cfc53d44f55 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Tue, 16 Jul 2024 15:52:38 -0700
Subject: [PATCH 13/15] Resolve ambiguity with LCP instead of block count with
multiple BFs at one neighbor hash
Created using spr 1.3.4
---
bolt/lib/Profile/YAMLProfileReader.cpp | 31 ++++++++++++++++++++------
1 file changed, 24 insertions(+), 7 deletions(-)
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 16d9b103d7a14..f979278b396e3 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -520,6 +520,16 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
size_t MatchedWithCallGraph = 0;
CallGraphMatcher CGMatcher(BC, YamlBP, IdToYamLBF);
+ ItaniumPartialDemangler Demangler;
+ auto GetBaseName = [&](std::string &FunctionName) {
+ if (Demangler.partialDemangle(FunctionName.c_str()))
+ return std::string("");
+ std::vector<char> Buffer(FunctionName.begin(), FunctionName.end());
+ size_t BufferSize;
+ char *BaseName = Demangler.getFunctionBaseName(&Buffer[0], &BufferSize);
+ return std::string(BaseName, BufferSize);
+ };
+
// Matches YAMLBF to BFs with neighbor hashes.
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (YamlBF.Used)
@@ -536,18 +546,25 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
CGMatcher.getBFsWithNeighborHash(Hash);
if (!BFsWithSameHash)
continue;
- // Finds the binary function with the closest block size to the profiled
+ // Finds the binary function with the longest common prefix to the profiled
// function and matches.
BinaryFunction *ClosestBF = nullptr;
- size_t MinDistance = std::numeric_limits<size_t>::max();
+ size_t LCP = 0;
+ std::string YamlBFBaseName = GetBaseName(YamlBF.Name);
for (BinaryFunction *BF : *BFsWithSameHash) {
if (ProfiledFunctions.count(BF))
continue;
- size_t Distance = YamlBF.NumBasicBlocks > BF->size()
- ? YamlBF.NumBasicBlocks - BF->size()
- : BF->size() - YamlBF.NumBasicBlocks;
- if (Distance < MinDistance) {
- MinDistance = Distance;
+ std::string BFName = std::string(BF->getOneName());
+ std::string BFBaseName = GetBaseName(BFName);
+ size_t PrefixLength = 0;
+ size_t N = std::min(YamlBFBaseName.size(), BFBaseName.size());
+ for (size_t I = 0; I < N; ++I) {
+ if (YamlBFBaseName[I] != BFBaseName[I])
+ break;
+ ++PrefixLength;
+ }
+ if (PrefixLength >= LCP) {
+ LCP = PrefixLength;
ClosestBF = BF;
}
}
>From 98247c73e9cf13661bcb0e82f8c21b8c77ec0967 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Wed, 17 Jul 2024 11:05:49 -0700
Subject: [PATCH 14/15] Fixed memory issue with buffer in getBaseName
Created using spr 1.3.4
---
bolt/lib/Profile/YAMLProfileReader.cpp | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index f979278b396e3..14e2e7f97bb8c 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -524,10 +524,14 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
auto GetBaseName = [&](std::string &FunctionName) {
if (Demangler.partialDemangle(FunctionName.c_str()))
return std::string("");
- std::vector<char> Buffer(FunctionName.begin(), FunctionName.end());
- size_t BufferSize;
- char *BaseName = Demangler.getFunctionBaseName(&Buffer[0], &BufferSize);
- return std::string(BaseName, BufferSize);
+ size_t BufferSize = 1;
+ char *Buffer = static_cast<char *>(std::malloc(BufferSize));
+ char *BaseName = Demangler.getFunctionBaseName(Buffer, &BufferSize);
+ if (!BaseName)
+ return std::string("");
+ std::string BaseNameStr(BaseName, BufferSize);
+ std::free(BaseName);
+ return BaseNameStr;
};
// Matches YAMLBF to BFs with neighbor hashes.
>From 3ac222d491d6c8849f1a0533e9c3fae0e9cc9f22 Mon Sep 17 00:00:00 2001
From: shawbyoung <shawbyoung at gmail.com>
Date: Wed, 17 Jul 2024 11:20:50 -0700
Subject: [PATCH 15/15] Change CallGraphMatchers to return optional instead of
pointer
Created using spr 1.3.4
---
bolt/include/bolt/Profile/YAMLProfileReader.h | 10 ++++++----
bolt/lib/Profile/YAMLProfileReader.cpp | 17 +++++++++--------
2 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/bolt/include/bolt/Profile/YAMLProfileReader.h b/bolt/include/bolt/Profile/YAMLProfileReader.h
index cef507e5a904e..bd5a86fd676a5 100644
--- a/bolt/include/bolt/Profile/YAMLProfileReader.h
+++ b/bolt/include/bolt/Profile/YAMLProfileReader.h
@@ -55,17 +55,19 @@ class YAMLProfileReader : public ProfileReaderBase {
ProfileLookupMap &IdToYAMLBF);
/// Returns the YamlBFs adjacent to the parameter YamlBF in the call graph.
- std::set<yaml::bolt::BinaryFunctionProfile *> *
+ std::optional<std::set<yaml::bolt::BinaryFunctionProfile *>>
getAdjacentYamlBFs(yaml::bolt::BinaryFunctionProfile &YamlBF) {
auto It = YamlBFAdjacencyMap.find(&YamlBF);
- return It == YamlBFAdjacencyMap.end() ? nullptr : &It->second;
+ return It == YamlBFAdjacencyMap.end() ? std::nullopt
+ : std::make_optional(It->second);
}
/// Returns the binary functions with the parameter neighbor hash.
- std::vector<BinaryFunction *> *
+ std::optional<std::vector<BinaryFunction *>>
getBFsWithNeighborHash(uint64_t NeighborHash) {
auto It = NeighborHashToBFs.find(NeighborHash);
- return It == NeighborHashToBFs.end() ? nullptr : &It->second;
+ return It == NeighborHashToBFs.end() ? std::nullopt
+ : std::make_optional(It->second);
}
private:
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index 14e2e7f97bb8c..16ca452850fb4 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -538,24 +538,25 @@ size_t YAMLProfileReader::matchWithCallGraph(BinaryContext &BC) {
for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
if (YamlBF.Used)
continue;
- std::set<yaml::bolt::BinaryFunctionProfile *> *AdjacentYamlBFs =
- CGMatcher.getAdjacentYamlBFs(YamlBF);
- if (!AdjacentYamlBFs)
+ auto AdjacentYamlBFsOpt = CGMatcher.getAdjacentYamlBFs(YamlBF);
+ if (!AdjacentYamlBFsOpt)
continue;
+ std::set<yaml::bolt::BinaryFunctionProfile *> AdjacentYamlBFs =
+ AdjacentYamlBFsOpt.value();
std::string AdjacentYamlBFsHashStr;
- for (auto *AdjacentYamlBF : *AdjacentYamlBFs)
+ for (auto *AdjacentYamlBF : AdjacentYamlBFs)
AdjacentYamlBFsHashStr += AdjacentYamlBF->Name;
uint64_t Hash = std::hash<std::string>{}(AdjacentYamlBFsHashStr);
- std::vector<BinaryFunction *> *BFsWithSameHash =
- CGMatcher.getBFsWithNeighborHash(Hash);
- if (!BFsWithSameHash)
+ auto BFsWithSameHashOpt = CGMatcher.getBFsWithNeighborHash(Hash);
+ if (!BFsWithSameHashOpt)
continue;
+ std::vector<BinaryFunction *> BFsWithSameHash = BFsWithSameHashOpt.value();
// Finds the binary function with the longest common prefix to the profiled
// function and matches.
BinaryFunction *ClosestBF = nullptr;
size_t LCP = 0;
std::string YamlBFBaseName = GetBaseName(YamlBF.Name);
- for (BinaryFunction *BF : *BFsWithSameHash) {
+ for (BinaryFunction *BF : BFsWithSameHash) {
if (ProfiledFunctions.count(BF))
continue;
std::string BFName = std::string(BF->getOneName());
More information about the llvm-branch-commits
mailing list