[llvm] Adding Matching and Inference Functionality to Propeller (PR #160706)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 25 06:38:04 PDT 2025
https://github.com/wdx727 created https://github.com/llvm/llvm-project/pull/160706
We have optimized the implementation of introducing the "matching and inference" technique into Propeller. In this new implementation, we have made every effort to avoid introducing new compilation parameters while ensuring compatibility with Propeller's current usage. Instead of creating a new profile format, we reused the existing one employed by Propeller. This new implementation is fully compatible with Propeller's current usage patterns and reduces the amount of code changes. For detailed information, please refer to the following RFC: https://discourse.llvm.org/t/rfc-adding-matching-and-inference-functionality-to-propeller/86238.
>From 963ca655afa61e03bbddd689755536a693253110 Mon Sep 17 00:00:00 2001
From: wudexin <wudexin at kuaishou.com>
Date: Wed, 24 Sep 2025 18:42:21 +0800
Subject: [PATCH] Adding Matching and Inference Functionality to Propeller
Reuse the original propeller symbol ordering file instead of regenerating it.
check
---
.../CodeGen/BasicBlockMatchingAndInference.h | 50 ++++++
.../CodeGen/BasicBlockSectionsProfileReader.h | 37 ++++
.../llvm/CodeGen/MachineBlockHashInfo.h | 106 +++++++++++
llvm/include/llvm/CodeGen/Passes.h | 7 +
llvm/include/llvm/InitializePasses.h | 2 +
llvm/include/llvm/Object/ELFTypes.h | 15 +-
llvm/include/llvm/ObjectYAML/ELFYAML.h | 1 +
.../Transforms/Utils/SampleProfileInference.h | 16 ++
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 14 +-
.../BasicBlockMatchingAndInference.cpp | 168 ++++++++++++++++++
llvm/lib/CodeGen/BasicBlockSections.cpp | 85 ++++++++-
.../BasicBlockSectionsProfileReader.cpp | 91 ++++++++++
llvm/lib/CodeGen/CMakeLists.txt | 2 +
llvm/lib/CodeGen/MachineBlockHashInfo.cpp | 111 ++++++++++++
llvm/lib/CodeGen/TargetPassConfig.cpp | 17 +-
llvm/lib/Object/ELF.cpp | 3 +-
llvm/lib/ObjectYAML/ELFEmitter.cpp | 3 +
llvm/lib/ObjectYAML/ELFYAML.cpp | 1 +
.../basic-block-address-map-with-bb-hash.ll | 86 +++++++++
...lock-sections-clusters-with-match-infer.ll | 90 ++++++++++
.../ELF/bb-addr-map-pgo-analysis-map.yaml | 5 +
llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml | 5 +
llvm/unittests/Object/ELFObjectFileTest.cpp | 118 ++++++++++++
llvm/unittests/Object/ELFTypesTest.cpp | 4 +-
24 files changed, 1023 insertions(+), 14 deletions(-)
create mode 100644 llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h
create mode 100644 llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
create mode 100644 llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
create mode 100644 llvm/lib/CodeGen/MachineBlockHashInfo.cpp
create mode 100644 llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll
create mode 100644 llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll
diff --git a/llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h b/llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h
new file mode 100644
index 0000000000000..66209d7685ecc
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h
@@ -0,0 +1,50 @@
+#ifndef LLVM_CODEGEN_BASIC_BLOCK_AND_INFERENCE_H
+#define LLVM_CODEGEN_BASIC_BLOCK_AND_INFERENCE_H
+
+#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Transforms/Utils/SampleProfileInference.h"
+
+namespace llvm {
+
+class BasicBlockMatchingAndInference : public MachineFunctionPass {
+private:
+ using Edge = std::pair<const MachineBasicBlock *, const MachineBasicBlock *>;
+ using BlockWeightMap = DenseMap<const MachineBasicBlock *, uint64_t>;
+ using EdgeWeightMap = DenseMap<Edge, uint64_t>;
+ using BlockEdgeMap = DenseMap<const MachineBasicBlock *,
+ SmallVector<const MachineBasicBlock *, 8>>;
+
+ struct WeightInfo {
+ // Weight of basic blocks.
+ BlockWeightMap BlockWeights;
+ // Weight of edges.
+ EdgeWeightMap EdgeWeights;
+ };
+
+public:
+ static char ID;
+ BasicBlockMatchingAndInference();
+
+ StringRef getPassName() const override {
+ return "Basic Block Matching and Inference";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+
+ std::optional<WeightInfo> getWeightInfo(StringRef FuncName) const;
+
+private:
+ StringMap<WeightInfo> ProgramWeightInfo;
+
+ WeightInfo initWeightInfoByMatching(MachineFunction &MF);
+
+ void generateWeightInfoByInference(MachineFunction &MF,
+ WeightInfo &MatchWeight);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_BASIC_BLOCK_AND_INFERENCE_H
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 08e6a0e3ef629..a27b921fb1205 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -31,6 +31,22 @@
namespace llvm {
+using Edge = std::pair<uint64_t, uint64_t>;
+using BlockWeightMap = DenseMap<uint64_t, uint64_t>;
+using EdgeWeightMap = DenseMap<Edge, uint64_t>;
+using BlockHashMap = DenseMap<uint64_t, uint64_t>;
+
+// This represents the weights of basic blocks and edges, and the hashed of
+// basic blocks for one function.
+struct WeightAndHashInfo {
+ // Weight of basic blocks.
+ BlockWeightMap BlockWeights;
+ // Weight of edges.
+ EdgeWeightMap EdgeWeights;
+ // Hashes of basic blocks.
+ BlockHashMap BlockHashes;
+};
+
// This struct represents the cluster information for a machine basic block,
// which is specifed by a unique ID (`MachineBasicBlock::BBID`).
struct BBClusterInfo {
@@ -98,6 +114,10 @@ class BasicBlockSectionsProfileReader {
SmallVector<SmallVector<unsigned>>
getClonePathsForFunction(StringRef FuncName) const;
+ // Returns the weight and hash info for the given function.
+ std::pair<bool, WeightAndHashInfo>
+ getWeightAndHashInfoForFunction(StringRef FuncName) const;
+
private:
StringRef getAliasName(StringRef FuncName) const {
auto R = FuncAliasMap.find(FuncName);
@@ -118,6 +138,16 @@ class BasicBlockSectionsProfileReader {
// positive integer.
Expected<UniqueBBID> parseUniqueBBID(StringRef S) const;
+ // Parses the weight of basic block and edgs.
+ Error parseWight(StringRef S, BlockWeightMap &BlockWeights,
+ EdgeWeightMap &EdgeWeights);
+
+ // Parses the hash of basic block.
+ Error parseBBHash(StringRef S, BlockHashMap &BlockHashes);
+
+ // Parse a pair in the form of "xxx:xxx"
+ Expected<std::pair<uint64_t, uint64_t>> parsePairItem(StringRef S) const;
+
// Reads the basic block sections profile for functions in this module.
Error ReadProfile();
@@ -146,6 +176,10 @@ class BasicBlockSectionsProfileReader {
// block in that cluster.
StringMap<FunctionPathAndClusterInfo> ProgramPathAndClusterInfo;
+ // This contains the weights of basic blocks and edges, and the hashes of
+ // basic blocks of the whole program.
+ StringMap<WeightAndHashInfo> ProgramWeightAndHashInfo;
+
// Some functions have alias names. We use this map to find the main alias
// name which appears in ProgramPathAndClusterInfo as a key.
StringMap<StringRef> FuncAliasMap;
@@ -204,6 +238,9 @@ class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass {
SmallVector<SmallVector<unsigned>>
getClonePathsForFunction(StringRef FuncName) const;
+ std::pair<bool, WeightAndHashInfo>
+ getWeightAndHashInfoForFunction(StringRef FuncName) const;
+
// Initializes the FunctionNameToDIFilename map for the current module and
// then reads the profile for the matching functions.
bool doInitialization(Module &M) override;
diff --git a/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
new file mode 100644
index 0000000000000..5de1b567e0309
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
@@ -0,0 +1,106 @@
+#ifndef LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
+#define LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+namespace llvm {
+
+/// An object wrapping several components of a basic block hash. The combined
+/// (blended) hash is represented and stored as one uint64_t, while individual
+/// components are of smaller size (e.g., uint16_t or uint8_t).
+struct BlendedBlockHash {
+private:
+ static uint64_t combineHashes(uint16_t Hash1, uint16_t Hash2, uint16_t Hash3,
+ uint16_t Hash4) {
+ uint64_t Hash = 0;
+
+ Hash |= uint64_t(Hash4);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash3);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash2);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash1);
+
+ return Hash;
+ }
+
+ static void parseHashes(uint64_t Hash, uint16_t &Hash1, uint16_t &Hash2,
+ uint16_t &Hash3, uint16_t &Hash4) {
+ Hash1 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash2 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash3 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash4 = Hash & 0xffff;
+ Hash >>= 16;
+ }
+
+public:
+ explicit BlendedBlockHash() {}
+
+ explicit BlendedBlockHash(uint64_t CombinedHash) {
+ parseHashes(CombinedHash, Offset, OpcodeHash, InstrHash, NeighborHash);
+ }
+
+ /// Combine the blended hash into uint64_t.
+ uint64_t combine() const {
+ return combineHashes(Offset, OpcodeHash, InstrHash, NeighborHash);
+ }
+
+ /// Compute a distance between two given blended hashes. The smaller the
+ /// distance, the more similar two blocks are. For identical basic blocks,
+ /// the distance is zero.
+ uint64_t distance(const BlendedBlockHash &BBH) const {
+ assert(OpcodeHash == BBH.OpcodeHash &&
+ "incorrect blended hash distance computation");
+ uint64_t Dist = 0;
+ // Account for NeighborHash
+ Dist += NeighborHash == BBH.NeighborHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for InstrHash
+ Dist += InstrHash == BBH.InstrHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for Offset
+ Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset);
+ return Dist;
+ }
+
+ /// The offset of the basic block from the function start.
+ uint16_t Offset{0};
+ /// (Loose) Hash of the basic block instructions, excluding operands.
+ uint16_t OpcodeHash{0};
+ /// (Strong) Hash of the basic block instructions, including opcodes and
+ /// operands.
+ uint16_t InstrHash{0};
+ /// Hash of the (loose) basic block together with (loose) hashes of its
+ /// successors and predecessors.
+ uint16_t NeighborHash{0};
+};
+
+class MachineBlockHashInfo : public MachineFunctionPass {
+ DenseMap<unsigned, uint64_t> MBBHashInfo;
+
+public:
+ static char ID;
+ MachineBlockHashInfo();
+
+ StringRef getPassName() const override { return "Basic Block Hash Compute"; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+
+ uint64_t getMBBHash(const MachineBasicBlock &MBB);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index d214ab9306c2f..063dd43e80638 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -67,6 +67,13 @@ namespace llvm {
MachineFunctionPass *createBasicBlockPathCloningPass();
+ /// createBasicBlockMatchingAndInferencePass - This pass enables matching
+ /// and inference when using propeller.
+ MachineFunctionPass *createBasicBlockMatchingAndInferencePass();
+
+ /// createMachineBlockHashInfoPass - This pass computes basic block hashes.
+ MachineFunctionPass *createMachineBlockHashInfoPass();
+
/// createMachineFunctionSplitterPass - This pass splits machine functions
/// using profile information.
MachineFunctionPass *createMachineFunctionSplitterPass();
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 1ce36a95317b4..3172b135426f6 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -53,6 +53,7 @@ void initializeAlwaysInlinerLegacyPassPass(PassRegistry &);
void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
void initializeAssumptionCacheTrackerPass(PassRegistry &);
void initializeAtomicExpandLegacyPass(PassRegistry &);
+void initializeBasicBlockMatchingAndInferencePass(PassRegistry &);
void initializeBasicBlockPathCloningPass(PassRegistry &);
void initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &);
void initializeBasicBlockSectionsPass(PassRegistry &);
@@ -185,6 +186,7 @@ void initializeMIRCanonicalizerPass(PassRegistry &);
void initializeMIRNamerPass(PassRegistry &);
void initializeMIRPrintingPassPass(PassRegistry &);
void initializeMachineBlockFrequencyInfoWrapperPassPass(PassRegistry &);
+void initializeMachineBlockHashInfoPass(PassRegistry&);
void initializeMachineBlockPlacementLegacyPass(PassRegistry &);
void initializeMachineBlockPlacementStatsLegacyPass(PassRegistry &);
void initializeMachineBranchProbabilityInfoWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index 87e4dbe448091..bbf07d87bb318 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -831,6 +831,7 @@ struct BBAddrMap {
bool BrProb : 1;
bool MultiBBRange : 1;
bool OmitBBEntries : 1;
+ bool BBHash : 1;
bool hasPGOAnalysis() const { return FuncEntryCount || BBFreq || BrProb; }
@@ -842,7 +843,8 @@ struct BBAddrMap {
(static_cast<uint8_t>(BBFreq) << 1) |
(static_cast<uint8_t>(BrProb) << 2) |
(static_cast<uint8_t>(MultiBBRange) << 3) |
- (static_cast<uint8_t>(OmitBBEntries) << 4);
+ (static_cast<uint8_t>(OmitBBEntries) << 4) |
+ (static_cast<uint8_t>(BBHash) << 5);
}
// Decodes from minimum bit width representation and validates no
@@ -851,7 +853,7 @@ struct BBAddrMap {
Features Feat{
static_cast<bool>(Val & (1 << 0)), static_cast<bool>(Val & (1 << 1)),
static_cast<bool>(Val & (1 << 2)), static_cast<bool>(Val & (1 << 3)),
- static_cast<bool>(Val & (1 << 4))};
+ static_cast<bool>(Val & (1 << 4)), static_cast<bool>(Val & (1 << 5))};
if (Feat.encode() != Val)
return createStringError(
std::error_code(), "invalid encoding for BBAddrMap::Features: 0x%x",
@@ -861,9 +863,9 @@ struct BBAddrMap {
bool operator==(const Features &Other) const {
return std::tie(FuncEntryCount, BBFreq, BrProb, MultiBBRange,
- OmitBBEntries) ==
+ OmitBBEntries, BBHash) ==
std::tie(Other.FuncEntryCount, Other.BBFreq, Other.BrProb,
- Other.MultiBBRange, Other.OmitBBEntries);
+ Other.MultiBBRange, Other.OmitBBEntries, Other.BBHash);
}
};
@@ -914,9 +916,10 @@ struct BBAddrMap {
uint32_t Size = 0; // Size of the basic block.
Metadata MD = {false, false, false, false,
false}; // Metdata for this basic block.
+ uint64_t Hash = 0; // Hash for this basic block.
- BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD)
- : ID(ID), Offset(Offset), Size(Size), MD(MD){};
+ BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD, uint64_t Hash = 0)
+ : ID(ID), Offset(Offset), Size(Size), MD(MD), Hash(Hash){};
bool operator==(const BBEntry &Other) const {
return ID == Other.ID && Offset == Other.Offset && Size == Other.Size &&
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index dfdfa055d65fa..9427042db4303 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -162,6 +162,7 @@ struct BBAddrMapEntry {
llvm::yaml::Hex64 AddressOffset;
llvm::yaml::Hex64 Size;
llvm::yaml::Hex64 Metadata;
+ llvm::yaml::Hex64 Hash;
};
uint8_t Version;
llvm::yaml::Hex8 Feature;
diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
index 7231e45fe8eb7..2b4db171bfdfb 100644
--- a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
+++ b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
@@ -130,6 +130,11 @@ template <typename FT> class SampleProfileInference {
SampleProfileInference(FunctionT &F, BlockEdgeMap &Successors,
BlockWeightMap &SampleBlockWeights)
: F(F), Successors(Successors), SampleBlockWeights(SampleBlockWeights) {}
+ SampleProfileInference(FunctionT &F, BlockEdgeMap &Successors,
+ BlockWeightMap &SampleBlockWeights,
+ EdgeWeightMap &SampleEdgeWeights)
+ : F(F), Successors(Successors), SampleBlockWeights(SampleBlockWeights),
+ SampleEdgeWeights(SampleEdgeWeights) {}
/// Apply the profile inference algorithm for a given function
void apply(BlockWeightMap &BlockWeights, EdgeWeightMap &EdgeWeights);
@@ -157,6 +162,9 @@ template <typename FT> class SampleProfileInference {
/// Map basic blocks to their sampled weights.
BlockWeightMap &SampleBlockWeights;
+
+ /// Map edges to their sampled weights.
+ EdgeWeightMap SampleEdgeWeights;
};
template <typename BT>
@@ -266,6 +274,14 @@ FlowFunction SampleProfileInference<BT>::createFlowFunction(
FlowJump Jump;
Jump.Source = BlockIndex[BB];
Jump.Target = BlockIndex[Succ];
+ auto It = SampleEdgeWeights.find(std::make_pair(BB, Succ));
+ if (It != SampleEdgeWeights.end()) {
+ Jump.HasUnknownWeight = false;
+ Jump.Weight = It->second;
+ } else {
+ Jump.HasUnknownWeight = true;
+ Jump.Weight = 0;
+ }
Func.Jumps.push_back(Jump);
}
}
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index bdcd54a135da9..41c084a4e4e49 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -40,6 +40,7 @@
#include "llvm/CodeGen/GCMetadataPrinter.h"
#include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineDominators.h"
@@ -180,6 +181,8 @@ static cl::opt<bool> PrintLatency(
cl::desc("Print instruction latencies as verbose asm comments"), cl::Hidden,
cl::init(false));
+extern cl::opt<bool> EmitBBHash;
+
STATISTIC(EmittedInsts, "Number of machine instrs printed");
char AsmPrinter::ID = 0;
@@ -454,6 +457,8 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<GCModuleInfo>();
AU.addRequired<LazyMachineBlockFrequencyInfoPass>();
AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
+ if (EmitBBHash)
+ AU.addRequired<MachineBlockHashInfo>();
}
bool AsmPrinter::doInitialization(Module &M) {
@@ -1419,7 +1424,8 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges) {
}
return {FuncEntryCountEnabled, BBFreqEnabled, BrProbEnabled,
MF.hasBBSections() && NumMBBSectionRanges > 1,
- static_cast<bool>(BBAddrMapSkipEmitBBEntries)};
+ static_cast<bool>(BBAddrMapSkipEmitBBEntries),
+ static_cast<bool>(EmitBBHash)};
}
void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
@@ -1477,6 +1483,8 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
PrevMBBEndSymbol = MBBSymbol;
}
+ auto MBHI = Features.BBHash ? &getAnalysis<MachineBlockHashInfo>() : nullptr;
+
if (!Features.OmitBBEntries) {
// TODO: Remove this check when version 1 is deprecated.
if (BBAddrMapVersion > 1) {
@@ -1496,6 +1504,10 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
emitLabelDifferenceAsULEB128(MBB.getEndSymbol(), MBBSymbol);
// Emit the Metadata.
OutStreamer->emitULEB128IntValue(getBBAddrMapMetadata(MBB));
+ // Emit the Hash.
+ if (MBHI) {
+ OutStreamer->emitULEB128IntValue(MBHI->getMBBHash(MBB));
+ }
}
PrevMBBEndSymbol = MBB.getEndSymbol();
diff --git a/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp b/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
new file mode 100644
index 0000000000000..e2776162043ff
--- /dev/null
+++ b/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
@@ -0,0 +1,168 @@
+#include "llvm/CodeGen/BasicBlockMatchingAndInference.h"
+#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/InitializePasses.h"
+#include <llvm/Support/CommandLine.h>
+
+using namespace llvm;
+
+static cl::opt<float>
+ PropellerInferThreshold("propeller-infer-threshold",
+ cl::desc("Threshold for infer stale profile"),
+ cl::init(0.6), cl::Optional);
+
+/// The object is used to identify and match basic blocks given their hashes.
+class StaleMatcher {
+public:
+ /// Initialize stale matcher.
+ void init(const std::vector<MachineBasicBlock *> &Blocks,
+ const std::vector<BlendedBlockHash> &Hashes) {
+ assert(Blocks.size() == Hashes.size() &&
+ "incorrect matcher initialization");
+ for (size_t I = 0; I < Blocks.size(); I++) {
+ MachineBasicBlock *Block = Blocks[I];
+ uint16_t OpHash = Hashes[I].OpcodeHash;
+ OpHashToBlocks[OpHash].push_back(std::make_pair(Hashes[I], Block));
+ }
+ }
+
+ /// Find the most similar block for a given hash.
+ MachineBasicBlock *matchBlock(BlendedBlockHash BlendedHash) const {
+ auto BlockIt = OpHashToBlocks.find(BlendedHash.OpcodeHash);
+ if (BlockIt == OpHashToBlocks.end()) {
+ return nullptr;
+ }
+ MachineBasicBlock *BestBlock = nullptr;
+ uint64_t BestDist = std::numeric_limits<uint64_t>::max();
+ for (auto It : BlockIt->second) {
+ MachineBasicBlock *Block = It.second;
+ BlendedBlockHash Hash = It.first;
+ uint64_t Dist = Hash.distance(BlendedHash);
+ if (BestBlock == nullptr || Dist < BestDist) {
+ BestDist = Dist;
+ BestBlock = Block;
+ }
+ }
+ return BestBlock;
+ }
+
+private:
+ using HashBlockPairType = std::pair<BlendedBlockHash, MachineBasicBlock *>;
+ std::unordered_map<uint16_t, std::vector<HashBlockPairType>> OpHashToBlocks;
+};
+
+INITIALIZE_PASS_BEGIN(BasicBlockMatchingAndInference,
+ "machine-block-match-infer",
+ "Machine Block Matching and Inference Analysis", true,
+ true)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockHashInfo)
+INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)
+INITIALIZE_PASS_END(BasicBlockMatchingAndInference, "machine-block-match-infer",
+ "Machine Block Matching and Inference Analysis", true, true)
+
+char BasicBlockMatchingAndInference::ID = 0;
+
+BasicBlockMatchingAndInference::BasicBlockMatchingAndInference()
+ : MachineFunctionPass(ID) {
+ initializeBasicBlockMatchingAndInferencePass(
+ *PassRegistry::getPassRegistry());
+}
+
+void BasicBlockMatchingAndInference::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<MachineBlockHashInfo>();
+ AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+std::optional<BasicBlockMatchingAndInference::WeightInfo>
+BasicBlockMatchingAndInference::getWeightInfo(StringRef FuncName) const {
+ auto It = ProgramWeightInfo.find(FuncName);
+ if (It == ProgramWeightInfo.end()) {
+ return std::nullopt;
+ }
+ return It->second;
+}
+
+BasicBlockMatchingAndInference::WeightInfo
+BasicBlockMatchingAndInference::initWeightInfoByMatching(MachineFunction &MF) {
+ std::vector<MachineBasicBlock *> Blocks;
+ std::vector<BlendedBlockHash> Hashes;
+ auto BSPR = &getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>();
+ auto MBHI = &getAnalysis<MachineBlockHashInfo>();
+ for (auto &Block : MF) {
+ Blocks.push_back(&Block);
+ Hashes.push_back(BlendedBlockHash(MBHI->getMBBHash(Block)));
+ }
+ StaleMatcher Matcher;
+ Matcher.init(Blocks, Hashes);
+ BasicBlockMatchingAndInference::WeightInfo MatchWeight;
+ auto [Flag, WeightAndHash] =
+ BSPR->getWeightAndHashInfoForFunction(MF.getName());
+ if (!Flag)
+ return std::move(MatchWeight);
+ for (auto &BlockWeight : WeightAndHash.BlockWeights) {
+ if (WeightAndHash.BlockHashes.count(BlockWeight.first)) {
+ auto Hash = WeightAndHash.BlockHashes[BlockWeight.first];
+ MachineBasicBlock *Block = Matcher.matchBlock(BlendedBlockHash(Hash));
+ if (Block != nullptr)
+ MatchWeight.BlockWeights.try_emplace(Block, BlockWeight.second);
+ }
+ }
+ for (auto &EdgeWeight : WeightAndHash.EdgeWeights) {
+ auto PredId = EdgeWeight.first.first;
+ auto SuccId = EdgeWeight.first.second;
+ if (WeightAndHash.BlockHashes.count(PredId) &&
+ WeightAndHash.BlockHashes.count(SuccId)) {
+ auto PredHash = WeightAndHash.BlockHashes[PredId];
+ auto SuccHash = WeightAndHash.BlockHashes[SuccId];
+ MachineBasicBlock *PredBlock =
+ Matcher.matchBlock(BlendedBlockHash(PredHash));
+ MachineBasicBlock *SuccBlock =
+ Matcher.matchBlock(BlendedBlockHash(SuccHash));
+ if (PredBlock != nullptr && SuccBlock != nullptr) {
+ MatchWeight.EdgeWeights.try_emplace(
+ std::make_pair(PredBlock, SuccBlock), EdgeWeight.second);
+ }
+ }
+ }
+ return std::move(MatchWeight);
+}
+
+void BasicBlockMatchingAndInference::generateWeightInfoByInference(
+ MachineFunction &MF,
+ BasicBlockMatchingAndInference::WeightInfo &MatchWeight) {
+ BlockEdgeMap Successors;
+ for (auto &Block : MF) {
+ for (auto *Succ : Block.successors())
+ Successors[&Block].push_back(Succ);
+ }
+ SampleProfileInference<MachineFunction> SPI(
+ MF, Successors, MatchWeight.BlockWeights, MatchWeight.EdgeWeights);
+ BlockWeightMap BlockWeights;
+ EdgeWeightMap EdgeWeights;
+ SPI.apply(BlockWeights, EdgeWeights);
+ ProgramWeightInfo.try_emplace(
+ MF.getName(), BasicBlockMatchingAndInference::WeightInfo{
+ std::move(BlockWeights), std::move(EdgeWeights)});
+}
+
+bool BasicBlockMatchingAndInference::runOnMachineFunction(MachineFunction &MF) {
+ if (MF.empty())
+ return false;
+ auto MatchWeight = initWeightInfoByMatching(MF);
+ // If the ratio of the number of MBBs in matching to the total number of MBBs
+ // in the function is less than the threshold value, the processing should be
+ // abandoned.
+ if (static_cast<float>(MatchWeight.BlockWeights.size()) / MF.size() <
+ PropellerInferThreshold) {
+ return false;
+ }
+ generateWeightInfoByInference(MF, MatchWeight);
+ return false;
+}
+
+MachineFunctionPass *llvm::createBasicBlockMatchingAndInferencePass() {
+ return new BasicBlockMatchingAndInference();
+}
diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp
index 1eedfc4b25912..5d5bd338ca137 100644
--- a/llvm/lib/CodeGen/BasicBlockSections.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSections.cpp
@@ -72,6 +72,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/CodeGen/BasicBlockMatchingAndInference.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
@@ -80,6 +81,7 @@
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/CodeLayout.h"
#include <optional>
using namespace llvm;
@@ -174,6 +176,77 @@ updateBranches(MachineFunction &MF,
}
}
+// This function generates the machine basic block clusters of "hot" blocks.
+// Currently, only "hot" basic blocks are supported for cluster creation.
+// TODO: Support multi-cluster creation and path cloning.
+static std::pair<bool, SmallVector<BBClusterInfo>>
+createBBClusterInfoForFunction(
+ const MachineFunction &MF,
+ BasicBlockMatchingAndInference *BMI) {
+ unsigned CurrentCluster = 0;
+ auto OptWeightInfo = BMI->getWeightInfo(MF.getName());
+ if (!OptWeightInfo)
+ return std::pair(false, SmallVector<BBClusterInfo>{});
+ auto BlockWeights = OptWeightInfo->BlockWeights;
+ auto EdgeWeights = OptWeightInfo->EdgeWeights;
+
+ SmallVector<const MachineBasicBlock *, 4> HotMBBs;
+ if (MF.size() <= 2) {
+ for (auto &MBB : MF) {
+ if (MBB.isEntryBlock() || BlockWeights[&MBB] > 0) {
+ HotMBBs.push_back(&MBB);
+ }
+ }
+ } else {
+ SmallVector<uint64_t, 0> BlockSizes(MF.size());
+ SmallVector<uint64_t, 0> BlockCounts(MF.size());
+ std::vector<const MachineBasicBlock *> OrigOrder;
+ OrigOrder.reserve(MF.size());
+ SmallVector<codelayout::EdgeCount, 0> JumpCounts;
+
+ // Init the MBB size and count.
+ for (auto &MBB : MF) {
+ auto NonDbgInsts =
+ instructionsWithoutDebug(MBB.instr_begin(), MBB.instr_end());
+ int NumInsts = std::distance(NonDbgInsts.begin(), NonDbgInsts.end());
+ BlockSizes[MBB.getNumber()] = 4 * NumInsts;
+ BlockCounts[MBB.getNumber()] = BlockWeights[&MBB];
+ OrigOrder.push_back(&MBB);
+ }
+
+ // Init the edge count.
+ for (auto &MBB : MF) {
+ for (auto *Succ : MBB.successors()) {
+ auto EdgeWeight = EdgeWeights[std::make_pair(&MBB, Succ)];
+ JumpCounts.push_back({static_cast<uint64_t>(MBB.getNumber()),
+ static_cast<uint64_t>(Succ->getNumber()),
+ EdgeWeight});
+ }
+ }
+
+ // Run the layout algorithm.
+ auto Result = computeExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
+ for (uint64_t R : Result) {
+ auto Block = OrigOrder[R];
+ if (Block->isEntryBlock() || BlockWeights[Block] > 0)
+ HotMBBs.push_back(Block);
+ }
+ }
+
+ // Generate the "hot" basic block cluster.
+ if(!HotMBBs.empty()) {
+ SmallVector<BBClusterInfo, 4> BBClusterInfos;
+ unsigned CurrentPosition = 0;
+ for (auto &MBB : HotMBBs) {
+ if (MBB->getBBID()) {
+ BBClusterInfos.push_back({*(MBB->getBBID()), CurrentCluster, CurrentPosition++});
+ }
+ }
+ return std::pair(true, std::move(BBClusterInfos));
+ }
+ return std::pair(false, SmallVector<BBClusterInfo>{});
+}
+
// This function sorts basic blocks according to the cluster's information.
// All explicitly specified clusters of basic blocks will be ordered
// accordingly. All non-specified BBs go into a separate "Cold" section.
@@ -313,12 +386,17 @@ bool BasicBlockSections::handleBBSections(MachineFunction &MF) {
DenseMap<UniqueBBID, BBClusterInfo> FuncClusterInfo;
if (BBSectionsType == BasicBlockSection::List) {
- auto [HasProfile, ClusterInfo] =
+ std::pair<bool, SmallVector<BBClusterInfo>> ExpClusterInfo;
+ if (auto *BMI = getAnalysisIfAvailable<BasicBlockMatchingAndInference>()) {
+ ExpClusterInfo = createBBClusterInfoForFunction(MF, BMI);
+ } else {
+ ExpClusterInfo =
getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>()
.getClusterInfoForFunction(MF.getName());
- if (!HasProfile)
+ }
+ if (!ExpClusterInfo.first)
return false;
- for (auto &BBClusterInfo : ClusterInfo) {
+ for (auto &BBClusterInfo : ExpClusterInfo.second) {
FuncClusterInfo.try_emplace(BBClusterInfo.BBID, BBClusterInfo);
}
}
@@ -401,6 +479,7 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
+ AU.addUsedIfAvailable<BasicBlockMatchingAndInference>();
AU.addUsedIfAvailable<MachineDominatorTreeWrapperPass>();
AU.addUsedIfAvailable<MachinePostDominatorTreeWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index fa54640265162..b3ba220f5bc9b 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -56,6 +56,53 @@ BasicBlockSectionsProfileReader::parseUniqueBBID(StringRef S) const {
static_cast<unsigned>(CloneID)};
}
+Error BasicBlockSectionsProfileReader::parseWight(StringRef S, BlockWeightMap &BlockWeights,
+ EdgeWeightMap &EdgeWeights) {
+ SmallVector<StringRef, 4> Values;
+ S.split(Values, ',');
+ auto BlockWeight = parsePairItem(Values[0]);
+ if (!BlockWeight)
+ return BlockWeight.takeError();
+ if (!BlockWeights.try_emplace(BlockWeight->first, BlockWeight->second).second)
+ return createProfileParseError(Twine("duplicate weight for basic block '") +
+ Twine(BlockWeight->first) + "'");
+ for (unsigned I = 1; I < Values.size(); ++I) {
+ auto EgdeWeight = parsePairItem(Values[I]);
+ if (!EgdeWeight)
+ return EgdeWeight.takeError();
+ auto Edge = std::make_pair(BlockWeight->first, EgdeWeight->first);
+ if (!EdgeWeights.try_emplace(Edge, EgdeWeight->second).second)
+ return createProfileParseError(Twine("duplicate weight for edge '(") +
+ Twine(BlockWeight->first) + "," +
+ Twine(EgdeWeight->first) +")'");
+ }
+ return Error::success();
+}
+
+Error BasicBlockSectionsProfileReader::parseBBHash(StringRef S, BlockHashMap &BlockHashes) {
+ auto BBHash = parsePairItem(S);
+ if (!BBHash)
+ return BBHash.takeError();
+ if (!BlockHashes.try_emplace(BBHash->first, BBHash->second).second)
+ return createProfileParseError(Twine("duplicate hash for basic block '") +
+ Twine(BBHash->first) + "'");
+ return Error::success();
+}
+
+Expected<std::pair<uint64_t, uint64_t>>
+BasicBlockSectionsProfileReader::parsePairItem(StringRef S) const {
+ auto Pos = S.find(':');
+ if (Pos == StringRef::npos || Pos == 0 || Pos == S.size() - 1)
+ return createProfileParseError(Twine("unable to parse pair item: '") +
+ S + "'");
+ unsigned long long First, Second;
+ if (getAsUnsignedInteger(S.substr(0, Pos), 10, First) ||
+ getAsUnsignedInteger(S.substr(Pos + 1), 10, Second))
+ return createProfileParseError(
+ Twine("unable to parse pair item: '") + S + "'");
+ return std::make_pair(First, Second);
+}
+
bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const {
return getClusterInfoForFunction(FuncName).first;
}
@@ -75,6 +122,15 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction(
return ProgramPathAndClusterInfo.lookup(getAliasName(FuncName)).ClonePaths;
}
+std::pair<bool, WeightAndHashInfo>
+BasicBlockSectionsProfileReader::getWeightAndHashInfoForFunction(
+ StringRef FuncName) const {
+ auto R = ProgramWeightAndHashInfo.find(getAliasName(FuncName));
+ return R != ProgramWeightAndHashInfo.end()
+ ? std::pair(true, R->second)
+ : std::pair(false, WeightAndHashInfo());
+}
+
// Reads the version 1 basic block sections profile. Profile for each function
// is encoded as follows:
// m <module_name>
@@ -132,6 +188,7 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction(
// ****************************************************************************
Error BasicBlockSectionsProfileReader::ReadV1Profile() {
auto FI = ProgramPathAndClusterInfo.end();
+ auto WI = ProgramWeightAndHashInfo.end();
// Current cluster ID corresponding to this function.
unsigned CurrentCluster = 0;
@@ -176,6 +233,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
// Skip the following profile by setting the profile iterator (FI) to
// the past-the-end element.
FI = ProgramPathAndClusterInfo.end();
+ WI = ProgramWeightAndHashInfo.end();
DIFilename = "";
continue;
}
@@ -193,6 +251,9 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
FI = R.first;
CurrentCluster = 0;
FuncBBIDs.clear();
+
+ WI = ProgramWeightAndHashInfo.try_emplace(Values.front()).first;
+
// We won't need DIFilename anymore. Clean it up to avoid its application
// on the next function.
DIFilename = "";
@@ -239,6 +300,30 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
}
continue;
}
+ case 'w': { // Basic block and edge weight specifier.
+ if ( WI == ProgramWeightAndHashInfo.end())
+ continue;
+ BlockWeightMap BlockWeights;
+ EdgeWeightMap EdgeWeights;
+ for (size_t I = 0; I < Values.size(); ++I) {
+ if (auto Err = parseWight(Values[I], BlockWeights, EdgeWeights))
+ return Err;
+ }
+ WI->second.BlockWeights = std::move(BlockWeights);
+ WI->second.EdgeWeights = std::move(EdgeWeights);
+ continue;
+ }
+ case 'h': { // Basic block hash secifier.
+ if ( WI == ProgramWeightAndHashInfo.end())
+ continue;
+ BlockHashMap BlockHashes;
+ for (size_t I = 0; I < Values.size(); ++I) {
+ if (auto Err = parseBBHash(Values[I], BlockHashes))
+ return Err;
+ }
+ WI->second.BlockHashes = std::move(BlockHashes);
+ continue;
+ }
default:
return createProfileParseError(Twine("invalid specifier: '") +
Twine(Specifier) + "'");
@@ -439,6 +524,12 @@ BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction(
return BBSPR.getClonePathsForFunction(FuncName);
}
+std::pair<bool, WeightAndHashInfo>
+BasicBlockSectionsProfileReaderWrapperPass::getWeightAndHashInfoForFunction(
+ StringRef FuncName) const {
+ return BBSPR.getWeightAndHashInfoForFunction(FuncName);
+}
+
BasicBlockSectionsProfileReader &
BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() {
return BBSPR;
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 5dd6413431255..1614fa0d41ba4 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -35,6 +35,7 @@ add_llvm_component_library(LLVMCodeGen
BasicBlockSections.cpp
BasicBlockPathCloning.cpp
BasicBlockSectionsProfileReader.cpp
+ BasicBlockMatchingAndInference.cpp
CalcSpillWeights.cpp
CallBrPrepare.cpp
CallingConvLower.cpp
@@ -108,6 +109,7 @@ add_llvm_component_library(LLVMCodeGen
LowerEmuTLS.cpp
MachineBasicBlock.cpp
MachineBlockFrequencyInfo.cpp
+ MachineBlockHashInfo.cpp
MachineBlockPlacement.cpp
MachineBranchProbabilityInfo.cpp
MachineCFGPrinter.cpp
diff --git a/llvm/lib/CodeGen/MachineBlockHashInfo.cpp b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp
new file mode 100644
index 0000000000000..a9636bd09a1a2
--- /dev/null
+++ b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp
@@ -0,0 +1,111 @@
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+using OperandHashFuncTy =
+ function_ref<uint64_t(uint64_t &, const MachineOperand &)>;
+
+uint64_t hashBlock(const MachineBasicBlock &MBB,
+ OperandHashFuncTy OperandHashFunc) {
+ uint64_t Hash = 0;
+ for (const MachineInstr &MI : MBB) {
+ if (MI.isMetaInstruction())
+ continue;
+ // Ignore terminator instruction
+ if (MI.isTerminator())
+ continue;
+ Hash = hashing::detail::hash_16_bytes(Hash, MI.getOpcode());
+ for (unsigned i = 0; i < MI.getNumOperands(); i++) {
+ Hash = OperandHashFunc(Hash, MI.getOperand(i));
+ }
+ }
+ return Hash;
+}
+
+/// Hashing a 64-bit integer to a 16-bit one.
+uint16_t hash_64_to_16(const uint64_t Hash) {
+ uint16_t Res = (uint16_t)(Hash & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 16) & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 32) & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 48) & 0xFFFF);
+ return Res;
+}
+
+uint64_t hashInstOperand(uint64_t &Hash, const MachineOperand &Operand) {
+ return hashing::detail::hash_16_bytes(Hash, hash_value(Operand));
+}
+
+INITIALIZE_PASS(MachineBlockHashInfo, "machine-block-hash",
+ "Machine Block Hash Analysis", true, true)
+
+char MachineBlockHashInfo::ID = 0;
+
+MachineBlockHashInfo::MachineBlockHashInfo() : MachineFunctionPass(ID) {
+ initializeMachineBlockHashInfoPass(*PassRegistry::getPassRegistry());
+}
+
+void MachineBlockHashInfo::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool MachineBlockHashInfo::runOnMachineFunction(MachineFunction &F) {
+ DenseMap<MachineBasicBlock *, BlendedBlockHash> BlendedHashes;
+ DenseMap<MachineBasicBlock *, uint64_t> OpcodeHashes;
+ uint16_t Offset = 0;
+ // Initialize hash components
+ for (MachineBasicBlock &MBB : F) {
+ BlendedBlockHash BlendedHash;
+ // offset of the machine basic block
+ BlendedHash.Offset = Offset;
+ Offset += MBB.size();
+ // Hashing opcodes
+ uint64_t OpcodeHash = hashBlock(
+ MBB, [](uint64_t &Hash, const MachineOperand &Op) { return Hash; });
+ OpcodeHashes[&MBB] = OpcodeHash;
+ BlendedHash.OpcodeHash = hash_64_to_16(OpcodeHash);
+ // Hash complete instructions
+ uint64_t InstrHash = hashBlock(MBB, hashInstOperand);
+ BlendedHash.InstrHash = hash_64_to_16(InstrHash);
+ BlendedHashes[&MBB] = BlendedHash;
+ }
+
+ // Initialize neighbor hash
+ for (MachineBasicBlock &MBB : F) {
+ uint64_t Hash = OpcodeHashes[&MBB];
+ // Append hashes of successors
+ for (MachineBasicBlock *SuccMBB : MBB.successors()) {
+ uint64_t SuccHash = OpcodeHashes[SuccMBB];
+ Hash = hashing::detail::hash_16_bytes(Hash, SuccHash);
+ }
+ // Append hashes of predecessors
+ for (MachineBasicBlock *PredMBB : MBB.predecessors()) {
+ uint64_t PredHash = OpcodeHashes[PredMBB];
+ Hash = hashing::detail::hash_16_bytes(Hash, PredHash);
+ }
+ BlendedHashes[&MBB].NeighborHash = hash_64_to_16(Hash);
+ }
+
+ // Assign hashes
+ for (MachineBasicBlock &MBB : F) {
+ if (MBB.getBBID()) {
+ MBBHashInfo[MBB.getBBID()->BaseID] = BlendedHashes[&MBB].combine();
+ }
+ }
+
+ return false;
+}
+
+uint64_t MachineBlockHashInfo::getMBBHash(const MachineBasicBlock &MBB) {
+ if (MBB.getBBID()) {
+ return MBBHashInfo[MBB.getBBID()->BaseID];
+ }
+ return 0;
+}
+
+MachineFunctionPass *llvm::createMachineBlockHashInfoPass() {
+ return new MachineBlockHashInfo();
+}
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 0095ce3d96277..9c3d9c28d18eb 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -266,6 +266,16 @@ static cl::opt<bool>
cl::desc("Split static data sections into hot and cold "
"sections using profile information"));
+/// Enable matching and inference when using propeller.
+static cl::opt<bool>
+ PropellerMatchInfer("propeller-match-infer",
+ cl::desc("Use match&infer to evaluate stale profile"),
+ cl::init(false), cl::Optional);
+
+cl::opt<bool> EmitBBHash("emit-bb-hash",
+ cl::desc("Emit the hash of basic block in the SHT_LLVM_BB_ADDR_MAP section."),
+ cl::init(false), cl::Optional);
+
/// Allow standard passes to be disabled by command line options. This supports
/// simple binary flags that either suppress the pass or do nothing.
/// i.e. -disable-mypass=false has no effect.
@@ -1275,10 +1285,15 @@ void TargetPassConfig::addMachinePasses() {
// address map (or both).
if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
TM->Options.BBAddrMap) {
+ if (EmitBBHash || PropellerMatchInfer)
+ addPass(llvm::createMachineBlockHashInfoPass());
if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(
TM->getBBSectionsFuncListBuf()));
- addPass(llvm::createBasicBlockPathCloningPass());
+ if (PropellerMatchInfer)
+ addPass(llvm::createBasicBlockMatchingAndInferencePass());
+ else
+ addPass(llvm::createBasicBlockPathCloningPass());
}
addPass(llvm::createBasicBlockSectionsPass());
}
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index bf42c92a242a1..c1faf6a401345 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -873,6 +873,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
uint32_t Offset = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
uint32_t MD = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
+ uint64_t Hash = FeatEnable.BBHash ? readULEB128As<uint64_t>(Data, Cur, ULEBSizeErr) : 0;
if (Version >= 1) {
// Offset is calculated relative to the end of the previous BB.
Offset += PrevBBEndOffset;
@@ -884,7 +885,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
MetadataDecodeErr = MetadataOrErr.takeError();
break;
}
- BBEntries.push_back({ID, Offset, Size, *MetadataOrErr});
+ BBEntries.push_back({ID, Offset, Size, *MetadataOrErr, Hash});
}
TotalNumBlocks += BBEntries.size();
}
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 9ae76a71ede5e..02e41c886f34d 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -1502,6 +1502,9 @@ void ELFState<ELFT>::writeSectionContent(
SHeader.sh_size += CBA.writeULEB128(BBE.AddressOffset);
SHeader.sh_size += CBA.writeULEB128(BBE.Size);
SHeader.sh_size += CBA.writeULEB128(BBE.Metadata);
+ if (FeatureOrErr->BBHash) {
+ SHeader.sh_size += CBA.writeULEB128(BBE.Hash);
+ }
}
}
if (!PGOAnalyses)
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 1f970739c1e7e..35001aca036ec 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -1872,6 +1872,7 @@ void MappingTraits<ELFYAML::BBAddrMapEntry::BBEntry>::mapping(
IO.mapRequired("AddressOffset", E.AddressOffset);
IO.mapRequired("Size", E.Size);
IO.mapRequired("Metadata", E.Metadata);
+ IO.mapOptional("Hash", E.Hash);
}
void MappingTraits<ELFYAML::PGOAnalysisMapEntry>::mapping(
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll
new file mode 100644
index 0000000000000..809621197dddf
--- /dev/null
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll
@@ -0,0 +1,86 @@
+; Check the basic block sections labels option when emit basic block hash.
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -emit-bb-hash | FileCheck %s --check-prefixes=CHECK,UNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-address-map -emit-bb-hash | FileCheck %s --check-prefixes=CHECK,NOUNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -split-machine-functions -emit-bb-hash | FileCheck %s --check-prefixes=CHECK,UNIQ
+
+define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 {
+ br i1 %0, label %3, label %8
+
+3:
+ %4 = invoke i32 @_Z3barv()
+ to label %8 unwind label %6
+ br label %10
+
+6:
+ landingpad { ptr, i32 }
+ catch ptr null
+ br label %12
+
+8:
+ %9 = call i32 @_Z3foov()
+ br i1 %1, label %12, label %10
+
+10:
+ %11 = select i1 %1, ptr blockaddress(@_Z3bazb, %3), ptr blockaddress(@_Z3bazb, %12) ; <ptr> [#uses=1]
+ indirectbr ptr %11, [label %3, label %12]
+
+12:
+ ret void
+}
+
+declare i32 @_Z3barv() #1
+
+declare i32 @_Z3foov() #1
+
+declare i32 @__gxx_personality_v0(...)
+
+; UNIQ: .section .text._Z3bazb,"ax", at progbits{{$}}
+; NOUNIQ: .section .text,"ax", at progbits,unique,1
+; CHECK-LABEL: _Z3bazb:
+; CHECK-LABEL: .Lfunc_begin0:
+; CHECK-LABEL: .LBB_END0_0:
+; CHECK-LABEL: .LBB0_1:
+; CHECK-LABEL: .LBB_END0_1:
+; CHECK-LABEL: .LBB0_2:
+; CHECK-LABEL: .LBB_END0_2:
+; CHECK-LABEL: .LBB0_3:
+; CHECK-LABEL: .LBB_END0_3:
+; CHECK-LABEL: .Lfunc_end0:
+
+; UNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3bazb{{$}}
+;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
+; NOUNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text,unique,1
+; CHECK-NEXT: .byte 2 # version
+; CHECK-NEXT: .byte 32 # feature
+; CHECK-NEXT: .quad .Lfunc_begin0 # function address
+; CHECK-NEXT: .byte 6 # number of basic blocks
+; CHECK-NEXT: .byte 0 # BB id
+; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0
+; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0
+; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 1 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_1-.LBB_END0_0
+; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1
+; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 3 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_2-.LBB_END0_1
+; CHECK-NEXT: .uleb128 .LBB_END0_2-.LBB0_2
+; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 4 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_3-.LBB_END0_2
+; CHECK-NEXT: .uleb128 .LBB_END0_3-.LBB0_3
+; CHECK-NEXT: .byte 16
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 5 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_4-.LBB_END0_3
+; CHECK-NEXT: .uleb128 .LBB_END0_4-.LBB0_4
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 2 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_5-.LBB_END0_4
+; CHECK-NEXT: .uleb128 .LBB_END0_5-.LBB0_5
+; CHECK-NEXT: .byte 5
+; CHECK-NEXT: .ascii "{{.*}}"
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll
new file mode 100644
index 0000000000000..8ac034f25940d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll
@@ -0,0 +1,90 @@
+; BB cluster section with -propeller-match-infer tests.
+;
+; Test1: Basic blocks #0 (entry), #1, and #3 will be placed in the same section.
+; The rest will be placed in the cold section.
+; RUN: echo 'v1' > %t1
+; RUN: echo 'f foo' >> %t1
+; RUN: echo 'c 0 1 3' >> %t1
+; RUN: echo 'w 0:100,1:100,2:0 1:100,3:100 2:0,3:0 3:100' >> %t1
+; RUN: echo 'h 0:10838971302264569856 1:10005270435075325958 2:10005130895882846216 3:17705462075698380809' >> %t1
+;
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -propeller-match-infer -basic-block-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
+;
+; Test2: Basic blocks #0 (entry), #2, and #3 will be placed in the same section.
+; The rest will be placed in the cold section.
+; RUN: echo 'v1' > %t2
+; RUN: echo 'f foo' >> %t2
+; RUN: echo 'c 0 2 3' >> %t2
+; RUN: echo 'w 0:100,1:0,2:100 1:0,3:0 2:100,3:100 3:100' >> %t2
+; RUN: echo 'h 0:10838971302264569856 1:10005270435075325958 2:10005130895882846216 3:17705462075698380809' >> %t2
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -propeller-match-infer -basic-block-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
+
+define void @foo(i1 zeroext) nounwind {
+ %2 = alloca i8, align 1
+ %3 = zext i1 %0 to i8
+ store i8 %3, ptr %2, align 1
+ %4 = load i8, ptr %2, align 1
+ %5 = trunc i8 %4 to i1
+ br i1 %5, label %6, label %8
+
+6: ; preds = %1
+ %7 = call i32 @bar()
+ br label %10
+
+8: ; preds = %1
+ %9 = call i32 @baz()
+ br label %10
+
+10: ; preds = %8, %6
+ ret void
+}
+
+declare i32 @bar() #1
+
+declare i32 @baz() #1
+
+; LINUX-SECTIONS1: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-LABEL: foo:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: # %bb.0:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: # %bb.1:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: .LBB0_3:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: .LBB_END0_3:
+; LINUX-SECTIONS1: .section .text.split.foo,"ax", at progbits
+; LINUX-SECTIONS1-LABEL: foo.cold:
+; LINUX-SECTIONS1-LABEL: .LBB_END0_2:
+; LINUX-SECTIONS1-NEXT: .size foo.cold, .LBB_END0_2-foo.cold
+; LINUX-SECTIONS1: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS1-LABEL: .Lfunc_end0:
+; LINUX-SECTIONS1-NEXT: .size foo, .Lfunc_end0-foo
+
+; LINUX-SECTIONS2: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-LABEL: foo:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: # %bb.0:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: # %bb.2:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: .LBB0_3:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: .LBB_END0_3:
+; LINUX-SECTIONS2: .section .text.split.foo,"ax", at progbits
+; LINUX-SECTIONS2-LABEL: foo.cold:
+; LINUX-SECTIONS2-LABEL: .LBB_END0_1:
+; LINUX-SECTIONS2-NEXT: .size foo.cold, .LBB_END0_1-foo.cold
+; LINUX-SECTIONS2: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS2-LABEL: .Lfunc_end0:
+; LINUX-SECTIONS2-NEXT: .size foo, .Lfunc_end0-foo
diff --git a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
index 299bf463cf4bc..93cf7da40c9c0 100644
--- a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
@@ -23,14 +23,17 @@
# VALID-NEXT: AddressOffset: 0x1
# VALID-NEXT: Size: 0x2
# VALID-NEXT: Metadata: 0x3
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 2
# VALID-NEXT: AddressOffset: 0x4
# VALID-NEXT: Size: 0x5
# VALID-NEXT: Metadata: 0x6
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 4
# VALID-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
# VALID-NEXT: Size: 0xFFFFFFFFFFFFFFF8
# VALID-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - Version: 2
# VALID-NEXT: Feature: 0xA
# VALID-NEXT: BBRanges:
@@ -40,6 +43,7 @@
# VALID-NEXT: AddressOffset: 0xA
# VALID-NEXT: Size: 0xB
# VALID-NEXT: Metadata: 0xC
+# VALID-NEXT: Hash: 0
# VALID-NEXT: PGOAnalyses:
# VALID-NEXT: - FuncEntryCount: 100
# VALID-NEXT: PGOBBEntries:
@@ -135,6 +139,7 @@ Sections:
# MULTI-NEXT: AddressOffset: 0x1
# MULTI-NEXT: Size: 0x2
# MULTI-NEXT: Metadata: 0x3
+# MULTI-NEXT: Hash: 0
# MULTI-NEXT: PGOAnalyses:
# MULTI-NEXT: - FuncEntryCount: 0
# MULTI-NEXT: PGOBBEntries:
diff --git a/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml b/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
index 8dbf97ef2bc12..aed4beff1d47c 100644
--- a/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
@@ -22,14 +22,17 @@
# VALID-NEXT: AddressOffset: 0x1
# VALID-NEXT: Size: 0x2
# VALID-NEXT: Metadata: 0x3
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 2
# VALID-NEXT: AddressOffset: 0x4
# VALID-NEXT: Size: 0x5
# VALID-NEXT: Metadata: 0x6
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 4
# VALID-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
# VALID-NEXT: Size: 0xFFFFFFFFFFFFFFF8
# VALID-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - Version: 2
# VALID-NEXT: Feature: 0x8
# VALID-NEXT: BBRanges:
@@ -39,6 +42,7 @@
# VALID-NEXT: AddressOffset: 0xA
# VALID-NEXT: Size: 0xB
# VALID-NEXT: Metadata: 0xC
+# VALID-NEXT: Hash: 0
--- !ELF
FileHeader:
@@ -126,6 +130,7 @@ Sections:
# MULTI-NEXT: AddressOffset: 0x1
# MULTI-NEXT: Size: 0x2
# MULTI-NEXT: Metadata: 0x3
+# MULTI-NEXT: Hash: 0
# MULTI-NEXT: - Name: '.llvm_bb_addr_map (1)'
# MULTI-NEXT: Type: SHT_LLVM_BB_ADDR_MAP
# MULTI-NEXT: Entries:
diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp
index 493e673d6a07d..91ecec891458d 100644
--- a/llvm/unittests/Object/ELFObjectFileTest.cpp
+++ b/llvm/unittests/Object/ELFObjectFileTest.cpp
@@ -977,6 +977,124 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
}
}
+// Test for the ELFObjectFile::readBBAddrMap API with BBHash.
+TEST(ELFObjectFileTest, ReadBBHash) {
+ StringRef CommonYamlString(R"(
+--- !ELF
+FileHeader:
+ Class: ELFCLASS64
+ Data: ELFDATA2LSB
+ Type: ET_EXEC
+Sections:
+ - Name: .llvm_bb_addr_map_1
+ Type: SHT_LLVM_BB_ADDR_MAP
+ Link: 1
+ Entries:
+ - Version: 2
+ Feature: 0x20
+ BBRanges:
+ - BaseAddress: 0x11111
+ BBEntries:
+ - ID: 1
+ AddressOffset: 0x0
+ Size: 0x1
+ Metadata: 0x2
+ Hash: 0x1
+ - Name: .llvm_bb_addr_map_2
+ Type: SHT_LLVM_BB_ADDR_MAP
+ Link: 1
+ Entries:
+ - Version: 2
+ Feature: 0x28
+ BBRanges:
+ - BaseAddress: 0x22222
+ BBEntries:
+ - ID: 2
+ AddressOffset: 0x0
+ Size: 0x2
+ Metadata: 0x4
+ Hash: 0x2
+ - BaseAddress: 0xFFFFF
+ BBEntries:
+ - ID: 15
+ AddressOffset: 0xF0
+ Size: 0xF1
+ Metadata: 0x1F
+ Hash: 0xF
+ - Name: .llvm_bb_addr_map_3
+ Type: SHT_LLVM_BB_ADDR_MAP
+ Link: 2
+ Entries:
+ - Version: 2
+ Feature: 0x20
+ BBRanges:
+ - BaseAddress: 0x33333
+ BBEntries:
+ - ID: 0
+ AddressOffset: 0x0
+ Size: 0x3
+ Metadata: 0x6
+ Hash: 0x3
+ - Name: .llvm_bb_addr_map_4
+ Type: SHT_LLVM_BB_ADDR_MAP
+ # Link: 0 (by default, can be overriden)
+ Entries:
+ - Version: 2
+ Feature: 0x20
+ BBRanges:
+ - BaseAddress: 0x44444
+ BBEntries:
+ - ID: 0
+ AddressOffset: 0x0
+ Size: 0x4
+ Metadata: 0x18
+ Hash: 0x4
+)");
+
+ BBAddrMap E1 = {
+ {{0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}, 0x1}}}}};
+ BBAddrMap E2 = {
+ {{0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}, 0x2}}},
+ {0xFFFFF, {{15, 0xF0, 0xF1, {true, true, true, true, true}, 0xF}}}}};
+ BBAddrMap E3 = {
+ {{0x33333, {{0, 0x0, 0x3, {false, true, true, false, false}, 0x3}}}}};
+ BBAddrMap E4 = {
+ {{0x44444, {{0, 0x0, 0x4, {false, false, false, true, true}, 0x4}}}}};
+
+ std::vector<BBAddrMap> Section0BBAddrMaps = {E4};
+ std::vector<BBAddrMap> Section1BBAddrMaps = {E3};
+ std::vector<BBAddrMap> Section2BBAddrMaps = {E1, E2};
+ std::vector<BBAddrMap> AllBBAddrMaps = {E1, E2, E3, E4};
+
+ auto DoCheckSucceeds = [&](StringRef YamlString,
+ std::optional<unsigned> TextSectionIndex,
+ std::vector<BBAddrMap> ExpectedResult) {
+ SCOPED_TRACE("for TextSectionIndex: " +
+ (TextSectionIndex ? llvm::Twine(*TextSectionIndex) : "{}") +
+ " and object yaml:\n" + YamlString);
+ SmallString<0> Storage;
+ Expected<ELFObjectFile<ELF64LE>> ElfOrErr =
+ toBinary<ELF64LE>(Storage, YamlString);
+ ASSERT_THAT_EXPECTED(ElfOrErr, Succeeded());
+
+ Expected<const typename ELF64LE::Shdr *> BBAddrMapSecOrErr =
+ ElfOrErr->getELFFile().getSection(1);
+ ASSERT_THAT_EXPECTED(BBAddrMapSecOrErr, Succeeded());
+ auto BBAddrMaps = ElfOrErr->readBBAddrMap(TextSectionIndex);
+ ASSERT_THAT_EXPECTED(BBAddrMaps, Succeeded());
+ EXPECT_EQ(*BBAddrMaps, ExpectedResult);
+ };
+
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/std::nullopt,
+ AllBBAddrMaps);
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/0,
+ Section0BBAddrMaps);
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/2,
+ Section1BBAddrMaps);
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/1,
+ Section2BBAddrMaps);
+}
+
// Test for the ELFObjectFile::readBBAddrMap API with PGOAnalysisMap.
TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
StringRef CommonYamlString(R"(
diff --git a/llvm/unittests/Object/ELFTypesTest.cpp b/llvm/unittests/Object/ELFTypesTest.cpp
index 13130dde80ef1..44cd8d397717a 100644
--- a/llvm/unittests/Object/ELFTypesTest.cpp
+++ b/llvm/unittests/Object/ELFTypesTest.cpp
@@ -125,9 +125,9 @@ TEST(ELFTypesTest, BBAddrMapFeaturesEncodingTest) {
TEST(ELFTypesTest, BBAddrMapFeaturesInvalidEncodingTest) {
const std::array<std::string, 2> Errors = {
- "invalid encoding for BBAddrMap::Features: 0x20",
+ "invalid encoding for BBAddrMap::Features: 0x40",
"invalid encoding for BBAddrMap::Features: 0xf0"};
- const std::array<uint8_t, 2> Values = {{0b10'0000, 0b1111'0000}};
+ const std::array<uint8_t, 2> Values = {{0b100'0000, 0b1111'0000}};
for (const auto &[Val, Error] : llvm::zip(Values, Errors)) {
EXPECT_THAT_ERROR(BBAddrMap::Features::decode(Val).takeError(),
FailedWithMessage(Error));
More information about the llvm-commits
mailing list