[llvm] Adding Matching and Inference Functionality to Propeller (PR #160706)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 26 01:40:44 PDT 2025
https://github.com/wdx727 updated https://github.com/llvm/llvm-project/pull/160706
>From 963ca655afa61e03bbddd689755536a693253110 Mon Sep 17 00:00:00 2001
From: wudexin <wudexin at kuaishou.com>
Date: Wed, 24 Sep 2025 18:42:21 +0800
Subject: [PATCH] Adding Matching and Inference Functionality to Propeller
Reuse the original propeller symbol ordering file instead of regenerating it.
check
---
.../CodeGen/BasicBlockMatchingAndInference.h | 50 ++++++
.../CodeGen/BasicBlockSectionsProfileReader.h | 37 ++++
.../llvm/CodeGen/MachineBlockHashInfo.h | 106 +++++++++++
llvm/include/llvm/CodeGen/Passes.h | 7 +
llvm/include/llvm/InitializePasses.h | 2 +
llvm/include/llvm/Object/ELFTypes.h | 15 +-
llvm/include/llvm/ObjectYAML/ELFYAML.h | 1 +
.../Transforms/Utils/SampleProfileInference.h | 16 ++
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 14 +-
.../BasicBlockMatchingAndInference.cpp | 168 ++++++++++++++++++
llvm/lib/CodeGen/BasicBlockSections.cpp | 85 ++++++++-
.../BasicBlockSectionsProfileReader.cpp | 91 ++++++++++
llvm/lib/CodeGen/CMakeLists.txt | 2 +
llvm/lib/CodeGen/MachineBlockHashInfo.cpp | 111 ++++++++++++
llvm/lib/CodeGen/TargetPassConfig.cpp | 17 +-
llvm/lib/Object/ELF.cpp | 3 +-
llvm/lib/ObjectYAML/ELFEmitter.cpp | 3 +
llvm/lib/ObjectYAML/ELFYAML.cpp | 1 +
.../basic-block-address-map-with-bb-hash.ll | 86 +++++++++
...lock-sections-clusters-with-match-infer.ll | 90 ++++++++++
.../ELF/bb-addr-map-pgo-analysis-map.yaml | 5 +
llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml | 5 +
llvm/unittests/Object/ELFObjectFileTest.cpp | 118 ++++++++++++
llvm/unittests/Object/ELFTypesTest.cpp | 4 +-
24 files changed, 1023 insertions(+), 14 deletions(-)
create mode 100644 llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h
create mode 100644 llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
create mode 100644 llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
create mode 100644 llvm/lib/CodeGen/MachineBlockHashInfo.cpp
create mode 100644 llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll
create mode 100644 llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll
diff --git a/llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h b/llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h
new file mode 100644
index 0000000000000..66209d7685ecc
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/BasicBlockMatchingAndInference.h
@@ -0,0 +1,50 @@
+#ifndef LLVM_CODEGEN_BASIC_BLOCK_AND_INFERENCE_H
+#define LLVM_CODEGEN_BASIC_BLOCK_AND_INFERENCE_H
+
+#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Transforms/Utils/SampleProfileInference.h"
+
+namespace llvm {
+
+class BasicBlockMatchingAndInference : public MachineFunctionPass {
+private:
+ using Edge = std::pair<const MachineBasicBlock *, const MachineBasicBlock *>;
+ using BlockWeightMap = DenseMap<const MachineBasicBlock *, uint64_t>;
+ using EdgeWeightMap = DenseMap<Edge, uint64_t>;
+ using BlockEdgeMap = DenseMap<const MachineBasicBlock *,
+ SmallVector<const MachineBasicBlock *, 8>>;
+
+ struct WeightInfo {
+ // Weight of basic blocks.
+ BlockWeightMap BlockWeights;
+ // Weight of edges.
+ EdgeWeightMap EdgeWeights;
+ };
+
+public:
+ static char ID;
+ BasicBlockMatchingAndInference();
+
+ StringRef getPassName() const override {
+ return "Basic Block Matching and Inference";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+
+ std::optional<WeightInfo> getWeightInfo(StringRef FuncName) const;
+
+private:
+ StringMap<WeightInfo> ProgramWeightInfo;
+
+ WeightInfo initWeightInfoByMatching(MachineFunction &MF);
+
+ void generateWeightInfoByInference(MachineFunction &MF,
+ WeightInfo &MatchWeight);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_BASIC_BLOCK_AND_INFERENCE_H
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 08e6a0e3ef629..a27b921fb1205 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -31,6 +31,22 @@
namespace llvm {
+using Edge = std::pair<uint64_t, uint64_t>;
+using BlockWeightMap = DenseMap<uint64_t, uint64_t>;
+using EdgeWeightMap = DenseMap<Edge, uint64_t>;
+using BlockHashMap = DenseMap<uint64_t, uint64_t>;
+
+// This represents the weights of basic blocks and edges, and the hashed of
+// basic blocks for one function.
+struct WeightAndHashInfo {
+ // Weight of basic blocks.
+ BlockWeightMap BlockWeights;
+ // Weight of edges.
+ EdgeWeightMap EdgeWeights;
+ // Hashes of basic blocks.
+ BlockHashMap BlockHashes;
+};
+
// This struct represents the cluster information for a machine basic block,
// which is specifed by a unique ID (`MachineBasicBlock::BBID`).
struct BBClusterInfo {
@@ -98,6 +114,10 @@ class BasicBlockSectionsProfileReader {
SmallVector<SmallVector<unsigned>>
getClonePathsForFunction(StringRef FuncName) const;
+ // Returns the weight and hash info for the given function.
+ std::pair<bool, WeightAndHashInfo>
+ getWeightAndHashInfoForFunction(StringRef FuncName) const;
+
private:
StringRef getAliasName(StringRef FuncName) const {
auto R = FuncAliasMap.find(FuncName);
@@ -118,6 +138,16 @@ class BasicBlockSectionsProfileReader {
// positive integer.
Expected<UniqueBBID> parseUniqueBBID(StringRef S) const;
+ // Parses the weight of basic block and edgs.
+ Error parseWight(StringRef S, BlockWeightMap &BlockWeights,
+ EdgeWeightMap &EdgeWeights);
+
+ // Parses the hash of basic block.
+ Error parseBBHash(StringRef S, BlockHashMap &BlockHashes);
+
+ // Parse a pair in the form of "xxx:xxx"
+ Expected<std::pair<uint64_t, uint64_t>> parsePairItem(StringRef S) const;
+
// Reads the basic block sections profile for functions in this module.
Error ReadProfile();
@@ -146,6 +176,10 @@ class BasicBlockSectionsProfileReader {
// block in that cluster.
StringMap<FunctionPathAndClusterInfo> ProgramPathAndClusterInfo;
+ // This contains the weights of basic blocks and edges, and the hashes of
+ // basic blocks of the whole program.
+ StringMap<WeightAndHashInfo> ProgramWeightAndHashInfo;
+
// Some functions have alias names. We use this map to find the main alias
// name which appears in ProgramPathAndClusterInfo as a key.
StringMap<StringRef> FuncAliasMap;
@@ -204,6 +238,9 @@ class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass {
SmallVector<SmallVector<unsigned>>
getClonePathsForFunction(StringRef FuncName) const;
+ std::pair<bool, WeightAndHashInfo>
+ getWeightAndHashInfoForFunction(StringRef FuncName) const;
+
// Initializes the FunctionNameToDIFilename map for the current module and
// then reads the profile for the matching functions.
bool doInitialization(Module &M) override;
diff --git a/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
new file mode 100644
index 0000000000000..5de1b567e0309
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
@@ -0,0 +1,106 @@
+#ifndef LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
+#define LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+namespace llvm {
+
+/// An object wrapping several components of a basic block hash. The combined
+/// (blended) hash is represented and stored as one uint64_t, while individual
+/// components are of smaller size (e.g., uint16_t or uint8_t).
+struct BlendedBlockHash {
+private:
+ static uint64_t combineHashes(uint16_t Hash1, uint16_t Hash2, uint16_t Hash3,
+ uint16_t Hash4) {
+ uint64_t Hash = 0;
+
+ Hash |= uint64_t(Hash4);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash3);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash2);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash1);
+
+ return Hash;
+ }
+
+ static void parseHashes(uint64_t Hash, uint16_t &Hash1, uint16_t &Hash2,
+ uint16_t &Hash3, uint16_t &Hash4) {
+ Hash1 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash2 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash3 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash4 = Hash & 0xffff;
+ Hash >>= 16;
+ }
+
+public:
+ explicit BlendedBlockHash() {}
+
+ explicit BlendedBlockHash(uint64_t CombinedHash) {
+ parseHashes(CombinedHash, Offset, OpcodeHash, InstrHash, NeighborHash);
+ }
+
+ /// Combine the blended hash into uint64_t.
+ uint64_t combine() const {
+ return combineHashes(Offset, OpcodeHash, InstrHash, NeighborHash);
+ }
+
+ /// Compute a distance between two given blended hashes. The smaller the
+ /// distance, the more similar two blocks are. For identical basic blocks,
+ /// the distance is zero.
+ uint64_t distance(const BlendedBlockHash &BBH) const {
+ assert(OpcodeHash == BBH.OpcodeHash &&
+ "incorrect blended hash distance computation");
+ uint64_t Dist = 0;
+ // Account for NeighborHash
+ Dist += NeighborHash == BBH.NeighborHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for InstrHash
+ Dist += InstrHash == BBH.InstrHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for Offset
+ Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset);
+ return Dist;
+ }
+
+ /// The offset of the basic block from the function start.
+ uint16_t Offset{0};
+ /// (Loose) Hash of the basic block instructions, excluding operands.
+ uint16_t OpcodeHash{0};
+ /// (Strong) Hash of the basic block instructions, including opcodes and
+ /// operands.
+ uint16_t InstrHash{0};
+ /// Hash of the (loose) basic block together with (loose) hashes of its
+ /// successors and predecessors.
+ uint16_t NeighborHash{0};
+};
+
+class MachineBlockHashInfo : public MachineFunctionPass {
+ DenseMap<unsigned, uint64_t> MBBHashInfo;
+
+public:
+ static char ID;
+ MachineBlockHashInfo();
+
+ StringRef getPassName() const override { return "Basic Block Hash Compute"; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+
+ uint64_t getMBBHash(const MachineBasicBlock &MBB);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index d214ab9306c2f..063dd43e80638 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -67,6 +67,13 @@ namespace llvm {
MachineFunctionPass *createBasicBlockPathCloningPass();
+ /// createBasicBlockMatchingAndInferencePass - This pass enables matching
+ /// and inference when using propeller.
+ MachineFunctionPass *createBasicBlockMatchingAndInferencePass();
+
+ /// createMachineBlockHashInfoPass - This pass computes basic block hashes.
+ MachineFunctionPass *createMachineBlockHashInfoPass();
+
/// createMachineFunctionSplitterPass - This pass splits machine functions
/// using profile information.
MachineFunctionPass *createMachineFunctionSplitterPass();
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 1ce36a95317b4..3172b135426f6 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -53,6 +53,7 @@ void initializeAlwaysInlinerLegacyPassPass(PassRegistry &);
void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
void initializeAssumptionCacheTrackerPass(PassRegistry &);
void initializeAtomicExpandLegacyPass(PassRegistry &);
+void initializeBasicBlockMatchingAndInferencePass(PassRegistry &);
void initializeBasicBlockPathCloningPass(PassRegistry &);
void initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &);
void initializeBasicBlockSectionsPass(PassRegistry &);
@@ -185,6 +186,7 @@ void initializeMIRCanonicalizerPass(PassRegistry &);
void initializeMIRNamerPass(PassRegistry &);
void initializeMIRPrintingPassPass(PassRegistry &);
void initializeMachineBlockFrequencyInfoWrapperPassPass(PassRegistry &);
+void initializeMachineBlockHashInfoPass(PassRegistry&);
void initializeMachineBlockPlacementLegacyPass(PassRegistry &);
void initializeMachineBlockPlacementStatsLegacyPass(PassRegistry &);
void initializeMachineBranchProbabilityInfoWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index 87e4dbe448091..bbf07d87bb318 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -831,6 +831,7 @@ struct BBAddrMap {
bool BrProb : 1;
bool MultiBBRange : 1;
bool OmitBBEntries : 1;
+ bool BBHash : 1;
bool hasPGOAnalysis() const { return FuncEntryCount || BBFreq || BrProb; }
@@ -842,7 +843,8 @@ struct BBAddrMap {
(static_cast<uint8_t>(BBFreq) << 1) |
(static_cast<uint8_t>(BrProb) << 2) |
(static_cast<uint8_t>(MultiBBRange) << 3) |
- (static_cast<uint8_t>(OmitBBEntries) << 4);
+ (static_cast<uint8_t>(OmitBBEntries) << 4) |
+ (static_cast<uint8_t>(BBHash) << 5);
}
// Decodes from minimum bit width representation and validates no
@@ -851,7 +853,7 @@ struct BBAddrMap {
Features Feat{
static_cast<bool>(Val & (1 << 0)), static_cast<bool>(Val & (1 << 1)),
static_cast<bool>(Val & (1 << 2)), static_cast<bool>(Val & (1 << 3)),
- static_cast<bool>(Val & (1 << 4))};
+ static_cast<bool>(Val & (1 << 4)), static_cast<bool>(Val & (1 << 5))};
if (Feat.encode() != Val)
return createStringError(
std::error_code(), "invalid encoding for BBAddrMap::Features: 0x%x",
@@ -861,9 +863,9 @@ struct BBAddrMap {
bool operator==(const Features &Other) const {
return std::tie(FuncEntryCount, BBFreq, BrProb, MultiBBRange,
- OmitBBEntries) ==
+ OmitBBEntries, BBHash) ==
std::tie(Other.FuncEntryCount, Other.BBFreq, Other.BrProb,
- Other.MultiBBRange, Other.OmitBBEntries);
+ Other.MultiBBRange, Other.OmitBBEntries, Other.BBHash);
}
};
@@ -914,9 +916,10 @@ struct BBAddrMap {
uint32_t Size = 0; // Size of the basic block.
Metadata MD = {false, false, false, false,
false}; // Metdata for this basic block.
+ uint64_t Hash = 0; // Hash for this basic block.
- BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD)
- : ID(ID), Offset(Offset), Size(Size), MD(MD){};
+ BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD, uint64_t Hash = 0)
+ : ID(ID), Offset(Offset), Size(Size), MD(MD), Hash(Hash){};
bool operator==(const BBEntry &Other) const {
return ID == Other.ID && Offset == Other.Offset && Size == Other.Size &&
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index dfdfa055d65fa..9427042db4303 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -162,6 +162,7 @@ struct BBAddrMapEntry {
llvm::yaml::Hex64 AddressOffset;
llvm::yaml::Hex64 Size;
llvm::yaml::Hex64 Metadata;
+ llvm::yaml::Hex64 Hash;
};
uint8_t Version;
llvm::yaml::Hex8 Feature;
diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
index 7231e45fe8eb7..2b4db171bfdfb 100644
--- a/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
+++ b/llvm/include/llvm/Transforms/Utils/SampleProfileInference.h
@@ -130,6 +130,11 @@ template <typename FT> class SampleProfileInference {
SampleProfileInference(FunctionT &F, BlockEdgeMap &Successors,
BlockWeightMap &SampleBlockWeights)
: F(F), Successors(Successors), SampleBlockWeights(SampleBlockWeights) {}
+ SampleProfileInference(FunctionT &F, BlockEdgeMap &Successors,
+ BlockWeightMap &SampleBlockWeights,
+ EdgeWeightMap &SampleEdgeWeights)
+ : F(F), Successors(Successors), SampleBlockWeights(SampleBlockWeights),
+ SampleEdgeWeights(SampleEdgeWeights) {}
/// Apply the profile inference algorithm for a given function
void apply(BlockWeightMap &BlockWeights, EdgeWeightMap &EdgeWeights);
@@ -157,6 +162,9 @@ template <typename FT> class SampleProfileInference {
/// Map basic blocks to their sampled weights.
BlockWeightMap &SampleBlockWeights;
+
+ /// Map edges to their sampled weights.
+ EdgeWeightMap SampleEdgeWeights;
};
template <typename BT>
@@ -266,6 +274,14 @@ FlowFunction SampleProfileInference<BT>::createFlowFunction(
FlowJump Jump;
Jump.Source = BlockIndex[BB];
Jump.Target = BlockIndex[Succ];
+ auto It = SampleEdgeWeights.find(std::make_pair(BB, Succ));
+ if (It != SampleEdgeWeights.end()) {
+ Jump.HasUnknownWeight = false;
+ Jump.Weight = It->second;
+ } else {
+ Jump.HasUnknownWeight = true;
+ Jump.Weight = 0;
+ }
Func.Jumps.push_back(Jump);
}
}
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index bdcd54a135da9..41c084a4e4e49 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -40,6 +40,7 @@
#include "llvm/CodeGen/GCMetadataPrinter.h"
#include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineDominators.h"
@@ -180,6 +181,8 @@ static cl::opt<bool> PrintLatency(
cl::desc("Print instruction latencies as verbose asm comments"), cl::Hidden,
cl::init(false));
+extern cl::opt<bool> EmitBBHash;
+
STATISTIC(EmittedInsts, "Number of machine instrs printed");
char AsmPrinter::ID = 0;
@@ -454,6 +457,8 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<GCModuleInfo>();
AU.addRequired<LazyMachineBlockFrequencyInfoPass>();
AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
+ if (EmitBBHash)
+ AU.addRequired<MachineBlockHashInfo>();
}
bool AsmPrinter::doInitialization(Module &M) {
@@ -1419,7 +1424,8 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges) {
}
return {FuncEntryCountEnabled, BBFreqEnabled, BrProbEnabled,
MF.hasBBSections() && NumMBBSectionRanges > 1,
- static_cast<bool>(BBAddrMapSkipEmitBBEntries)};
+ static_cast<bool>(BBAddrMapSkipEmitBBEntries),
+ static_cast<bool>(EmitBBHash)};
}
void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
@@ -1477,6 +1483,8 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
PrevMBBEndSymbol = MBBSymbol;
}
+ auto MBHI = Features.BBHash ? &getAnalysis<MachineBlockHashInfo>() : nullptr;
+
if (!Features.OmitBBEntries) {
// TODO: Remove this check when version 1 is deprecated.
if (BBAddrMapVersion > 1) {
@@ -1496,6 +1504,10 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
emitLabelDifferenceAsULEB128(MBB.getEndSymbol(), MBBSymbol);
// Emit the Metadata.
OutStreamer->emitULEB128IntValue(getBBAddrMapMetadata(MBB));
+ // Emit the Hash.
+ if (MBHI) {
+ OutStreamer->emitULEB128IntValue(MBHI->getMBBHash(MBB));
+ }
}
PrevMBBEndSymbol = MBB.getEndSymbol();
diff --git a/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp b/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
new file mode 100644
index 0000000000000..e2776162043ff
--- /dev/null
+++ b/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
@@ -0,0 +1,168 @@
+#include "llvm/CodeGen/BasicBlockMatchingAndInference.h"
+#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/InitializePasses.h"
+#include <llvm/Support/CommandLine.h>
+
+using namespace llvm;
+
+static cl::opt<float>
+ PropellerInferThreshold("propeller-infer-threshold",
+ cl::desc("Threshold for infer stale profile"),
+ cl::init(0.6), cl::Optional);
+
+/// The object is used to identify and match basic blocks given their hashes.
+class StaleMatcher {
+public:
+ /// Initialize stale matcher.
+ void init(const std::vector<MachineBasicBlock *> &Blocks,
+ const std::vector<BlendedBlockHash> &Hashes) {
+ assert(Blocks.size() == Hashes.size() &&
+ "incorrect matcher initialization");
+ for (size_t I = 0; I < Blocks.size(); I++) {
+ MachineBasicBlock *Block = Blocks[I];
+ uint16_t OpHash = Hashes[I].OpcodeHash;
+ OpHashToBlocks[OpHash].push_back(std::make_pair(Hashes[I], Block));
+ }
+ }
+
+ /// Find the most similar block for a given hash.
+ MachineBasicBlock *matchBlock(BlendedBlockHash BlendedHash) const {
+ auto BlockIt = OpHashToBlocks.find(BlendedHash.OpcodeHash);
+ if (BlockIt == OpHashToBlocks.end()) {
+ return nullptr;
+ }
+ MachineBasicBlock *BestBlock = nullptr;
+ uint64_t BestDist = std::numeric_limits<uint64_t>::max();
+ for (auto It : BlockIt->second) {
+ MachineBasicBlock *Block = It.second;
+ BlendedBlockHash Hash = It.first;
+ uint64_t Dist = Hash.distance(BlendedHash);
+ if (BestBlock == nullptr || Dist < BestDist) {
+ BestDist = Dist;
+ BestBlock = Block;
+ }
+ }
+ return BestBlock;
+ }
+
+private:
+ using HashBlockPairType = std::pair<BlendedBlockHash, MachineBasicBlock *>;
+ std::unordered_map<uint16_t, std::vector<HashBlockPairType>> OpHashToBlocks;
+};
+
+INITIALIZE_PASS_BEGIN(BasicBlockMatchingAndInference,
+ "machine-block-match-infer",
+ "Machine Block Matching and Inference Analysis", true,
+ true)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockHashInfo)
+INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)
+INITIALIZE_PASS_END(BasicBlockMatchingAndInference, "machine-block-match-infer",
+ "Machine Block Matching and Inference Analysis", true, true)
+
+char BasicBlockMatchingAndInference::ID = 0;
+
+BasicBlockMatchingAndInference::BasicBlockMatchingAndInference()
+ : MachineFunctionPass(ID) {
+ initializeBasicBlockMatchingAndInferencePass(
+ *PassRegistry::getPassRegistry());
+}
+
+void BasicBlockMatchingAndInference::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<MachineBlockHashInfo>();
+ AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+std::optional<BasicBlockMatchingAndInference::WeightInfo>
+BasicBlockMatchingAndInference::getWeightInfo(StringRef FuncName) const {
+ auto It = ProgramWeightInfo.find(FuncName);
+ if (It == ProgramWeightInfo.end()) {
+ return std::nullopt;
+ }
+ return It->second;
+}
+
+BasicBlockMatchingAndInference::WeightInfo
+BasicBlockMatchingAndInference::initWeightInfoByMatching(MachineFunction &MF) {
+ std::vector<MachineBasicBlock *> Blocks;
+ std::vector<BlendedBlockHash> Hashes;
+ auto BSPR = &getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>();
+ auto MBHI = &getAnalysis<MachineBlockHashInfo>();
+ for (auto &Block : MF) {
+ Blocks.push_back(&Block);
+ Hashes.push_back(BlendedBlockHash(MBHI->getMBBHash(Block)));
+ }
+ StaleMatcher Matcher;
+ Matcher.init(Blocks, Hashes);
+ BasicBlockMatchingAndInference::WeightInfo MatchWeight;
+ auto [Flag, WeightAndHash] =
+ BSPR->getWeightAndHashInfoForFunction(MF.getName());
+ if (!Flag)
+ return std::move(MatchWeight);
+ for (auto &BlockWeight : WeightAndHash.BlockWeights) {
+ if (WeightAndHash.BlockHashes.count(BlockWeight.first)) {
+ auto Hash = WeightAndHash.BlockHashes[BlockWeight.first];
+ MachineBasicBlock *Block = Matcher.matchBlock(BlendedBlockHash(Hash));
+ if (Block != nullptr)
+ MatchWeight.BlockWeights.try_emplace(Block, BlockWeight.second);
+ }
+ }
+ for (auto &EdgeWeight : WeightAndHash.EdgeWeights) {
+ auto PredId = EdgeWeight.first.first;
+ auto SuccId = EdgeWeight.first.second;
+ if (WeightAndHash.BlockHashes.count(PredId) &&
+ WeightAndHash.BlockHashes.count(SuccId)) {
+ auto PredHash = WeightAndHash.BlockHashes[PredId];
+ auto SuccHash = WeightAndHash.BlockHashes[SuccId];
+ MachineBasicBlock *PredBlock =
+ Matcher.matchBlock(BlendedBlockHash(PredHash));
+ MachineBasicBlock *SuccBlock =
+ Matcher.matchBlock(BlendedBlockHash(SuccHash));
+ if (PredBlock != nullptr && SuccBlock != nullptr) {
+ MatchWeight.EdgeWeights.try_emplace(
+ std::make_pair(PredBlock, SuccBlock), EdgeWeight.second);
+ }
+ }
+ }
+ return std::move(MatchWeight);
+}
+
+void BasicBlockMatchingAndInference::generateWeightInfoByInference(
+ MachineFunction &MF,
+ BasicBlockMatchingAndInference::WeightInfo &MatchWeight) {
+ BlockEdgeMap Successors;
+ for (auto &Block : MF) {
+ for (auto *Succ : Block.successors())
+ Successors[&Block].push_back(Succ);
+ }
+ SampleProfileInference<MachineFunction> SPI(
+ MF, Successors, MatchWeight.BlockWeights, MatchWeight.EdgeWeights);
+ BlockWeightMap BlockWeights;
+ EdgeWeightMap EdgeWeights;
+ SPI.apply(BlockWeights, EdgeWeights);
+ ProgramWeightInfo.try_emplace(
+ MF.getName(), BasicBlockMatchingAndInference::WeightInfo{
+ std::move(BlockWeights), std::move(EdgeWeights)});
+}
+
+bool BasicBlockMatchingAndInference::runOnMachineFunction(MachineFunction &MF) {
+ if (MF.empty())
+ return false;
+ auto MatchWeight = initWeightInfoByMatching(MF);
+ // If the ratio of the number of MBBs in matching to the total number of MBBs
+ // in the function is less than the threshold value, the processing should be
+ // abandoned.
+ if (static_cast<float>(MatchWeight.BlockWeights.size()) / MF.size() <
+ PropellerInferThreshold) {
+ return false;
+ }
+ generateWeightInfoByInference(MF, MatchWeight);
+ return false;
+}
+
+MachineFunctionPass *llvm::createBasicBlockMatchingAndInferencePass() {
+ return new BasicBlockMatchingAndInference();
+}
diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp
index 1eedfc4b25912..5d5bd338ca137 100644
--- a/llvm/lib/CodeGen/BasicBlockSections.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSections.cpp
@@ -72,6 +72,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/CodeGen/BasicBlockMatchingAndInference.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
@@ -80,6 +81,7 @@
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/CodeLayout.h"
#include <optional>
using namespace llvm;
@@ -174,6 +176,77 @@ updateBranches(MachineFunction &MF,
}
}
+// This function generates the machine basic block clusters of "hot" blocks.
+// Currently, only "hot" basic blocks are supported for cluster creation.
+// TODO: Support multi-cluster creation and path cloning.
+static std::pair<bool, SmallVector<BBClusterInfo>>
+createBBClusterInfoForFunction(
+ const MachineFunction &MF,
+ BasicBlockMatchingAndInference *BMI) {
+ unsigned CurrentCluster = 0;
+ auto OptWeightInfo = BMI->getWeightInfo(MF.getName());
+ if (!OptWeightInfo)
+ return std::pair(false, SmallVector<BBClusterInfo>{});
+ auto BlockWeights = OptWeightInfo->BlockWeights;
+ auto EdgeWeights = OptWeightInfo->EdgeWeights;
+
+ SmallVector<const MachineBasicBlock *, 4> HotMBBs;
+ if (MF.size() <= 2) {
+ for (auto &MBB : MF) {
+ if (MBB.isEntryBlock() || BlockWeights[&MBB] > 0) {
+ HotMBBs.push_back(&MBB);
+ }
+ }
+ } else {
+ SmallVector<uint64_t, 0> BlockSizes(MF.size());
+ SmallVector<uint64_t, 0> BlockCounts(MF.size());
+ std::vector<const MachineBasicBlock *> OrigOrder;
+ OrigOrder.reserve(MF.size());
+ SmallVector<codelayout::EdgeCount, 0> JumpCounts;
+
+ // Init the MBB size and count.
+ for (auto &MBB : MF) {
+ auto NonDbgInsts =
+ instructionsWithoutDebug(MBB.instr_begin(), MBB.instr_end());
+ int NumInsts = std::distance(NonDbgInsts.begin(), NonDbgInsts.end());
+ BlockSizes[MBB.getNumber()] = 4 * NumInsts;
+ BlockCounts[MBB.getNumber()] = BlockWeights[&MBB];
+ OrigOrder.push_back(&MBB);
+ }
+
+ // Init the edge count.
+ for (auto &MBB : MF) {
+ for (auto *Succ : MBB.successors()) {
+ auto EdgeWeight = EdgeWeights[std::make_pair(&MBB, Succ)];
+ JumpCounts.push_back({static_cast<uint64_t>(MBB.getNumber()),
+ static_cast<uint64_t>(Succ->getNumber()),
+ EdgeWeight});
+ }
+ }
+
+ // Run the layout algorithm.
+ auto Result = computeExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
+ for (uint64_t R : Result) {
+ auto Block = OrigOrder[R];
+ if (Block->isEntryBlock() || BlockWeights[Block] > 0)
+ HotMBBs.push_back(Block);
+ }
+ }
+
+ // Generate the "hot" basic block cluster.
+ if(!HotMBBs.empty()) {
+ SmallVector<BBClusterInfo, 4> BBClusterInfos;
+ unsigned CurrentPosition = 0;
+ for (auto &MBB : HotMBBs) {
+ if (MBB->getBBID()) {
+ BBClusterInfos.push_back({*(MBB->getBBID()), CurrentCluster, CurrentPosition++});
+ }
+ }
+ return std::pair(true, std::move(BBClusterInfos));
+ }
+ return std::pair(false, SmallVector<BBClusterInfo>{});
+}
+
// This function sorts basic blocks according to the cluster's information.
// All explicitly specified clusters of basic blocks will be ordered
// accordingly. All non-specified BBs go into a separate "Cold" section.
@@ -313,12 +386,17 @@ bool BasicBlockSections::handleBBSections(MachineFunction &MF) {
DenseMap<UniqueBBID, BBClusterInfo> FuncClusterInfo;
if (BBSectionsType == BasicBlockSection::List) {
- auto [HasProfile, ClusterInfo] =
+ std::pair<bool, SmallVector<BBClusterInfo>> ExpClusterInfo;
+ if (auto *BMI = getAnalysisIfAvailable<BasicBlockMatchingAndInference>()) {
+ ExpClusterInfo = createBBClusterInfoForFunction(MF, BMI);
+ } else {
+ ExpClusterInfo =
getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>()
.getClusterInfoForFunction(MF.getName());
- if (!HasProfile)
+ }
+ if (!ExpClusterInfo.first)
return false;
- for (auto &BBClusterInfo : ClusterInfo) {
+ for (auto &BBClusterInfo : ExpClusterInfo.second) {
FuncClusterInfo.try_emplace(BBClusterInfo.BBID, BBClusterInfo);
}
}
@@ -401,6 +479,7 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
+ AU.addUsedIfAvailable<BasicBlockMatchingAndInference>();
AU.addUsedIfAvailable<MachineDominatorTreeWrapperPass>();
AU.addUsedIfAvailable<MachinePostDominatorTreeWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index fa54640265162..b3ba220f5bc9b 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -56,6 +56,53 @@ BasicBlockSectionsProfileReader::parseUniqueBBID(StringRef S) const {
static_cast<unsigned>(CloneID)};
}
+Error BasicBlockSectionsProfileReader::parseWight(StringRef S, BlockWeightMap &BlockWeights,
+ EdgeWeightMap &EdgeWeights) {
+ SmallVector<StringRef, 4> Values;
+ S.split(Values, ',');
+ auto BlockWeight = parsePairItem(Values[0]);
+ if (!BlockWeight)
+ return BlockWeight.takeError();
+ if (!BlockWeights.try_emplace(BlockWeight->first, BlockWeight->second).second)
+ return createProfileParseError(Twine("duplicate weight for basic block '") +
+ Twine(BlockWeight->first) + "'");
+ for (unsigned I = 1; I < Values.size(); ++I) {
+ auto EgdeWeight = parsePairItem(Values[I]);
+ if (!EgdeWeight)
+ return EgdeWeight.takeError();
+ auto Edge = std::make_pair(BlockWeight->first, EgdeWeight->first);
+ if (!EdgeWeights.try_emplace(Edge, EgdeWeight->second).second)
+ return createProfileParseError(Twine("duplicate weight for edge '(") +
+ Twine(BlockWeight->first) + "," +
+ Twine(EgdeWeight->first) +")'");
+ }
+ return Error::success();
+}
+
+Error BasicBlockSectionsProfileReader::parseBBHash(StringRef S, BlockHashMap &BlockHashes) {
+ auto BBHash = parsePairItem(S);
+ if (!BBHash)
+ return BBHash.takeError();
+ if (!BlockHashes.try_emplace(BBHash->first, BBHash->second).second)
+ return createProfileParseError(Twine("duplicate hash for basic block '") +
+ Twine(BBHash->first) + "'");
+ return Error::success();
+}
+
+Expected<std::pair<uint64_t, uint64_t>>
+BasicBlockSectionsProfileReader::parsePairItem(StringRef S) const {
+ auto Pos = S.find(':');
+ if (Pos == StringRef::npos || Pos == 0 || Pos == S.size() - 1)
+ return createProfileParseError(Twine("unable to parse pair item: '") +
+ S + "'");
+ unsigned long long First, Second;
+ if (getAsUnsignedInteger(S.substr(0, Pos), 10, First) ||
+ getAsUnsignedInteger(S.substr(Pos + 1), 10, Second))
+ return createProfileParseError(
+ Twine("unable to parse pair item: '") + S + "'");
+ return std::make_pair(First, Second);
+}
+
bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const {
return getClusterInfoForFunction(FuncName).first;
}
@@ -75,6 +122,15 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction(
return ProgramPathAndClusterInfo.lookup(getAliasName(FuncName)).ClonePaths;
}
+std::pair<bool, WeightAndHashInfo>
+BasicBlockSectionsProfileReader::getWeightAndHashInfoForFunction(
+ StringRef FuncName) const {
+ auto R = ProgramWeightAndHashInfo.find(getAliasName(FuncName));
+ return R != ProgramWeightAndHashInfo.end()
+ ? std::pair(true, R->second)
+ : std::pair(false, WeightAndHashInfo());
+}
+
// Reads the version 1 basic block sections profile. Profile for each function
// is encoded as follows:
// m <module_name>
@@ -132,6 +188,7 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction(
// ****************************************************************************
Error BasicBlockSectionsProfileReader::ReadV1Profile() {
auto FI = ProgramPathAndClusterInfo.end();
+ auto WI = ProgramWeightAndHashInfo.end();
// Current cluster ID corresponding to this function.
unsigned CurrentCluster = 0;
@@ -176,6 +233,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
// Skip the following profile by setting the profile iterator (FI) to
// the past-the-end element.
FI = ProgramPathAndClusterInfo.end();
+ WI = ProgramWeightAndHashInfo.end();
DIFilename = "";
continue;
}
@@ -193,6 +251,9 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
FI = R.first;
CurrentCluster = 0;
FuncBBIDs.clear();
+
+ WI = ProgramWeightAndHashInfo.try_emplace(Values.front()).first;
+
// We won't need DIFilename anymore. Clean it up to avoid its application
// on the next function.
DIFilename = "";
@@ -239,6 +300,30 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
}
continue;
}
+ case 'w': { // Basic block and edge weight specifier.
+ if ( WI == ProgramWeightAndHashInfo.end())
+ continue;
+ BlockWeightMap BlockWeights;
+ EdgeWeightMap EdgeWeights;
+ for (size_t I = 0; I < Values.size(); ++I) {
+ if (auto Err = parseWight(Values[I], BlockWeights, EdgeWeights))
+ return Err;
+ }
+ WI->second.BlockWeights = std::move(BlockWeights);
+ WI->second.EdgeWeights = std::move(EdgeWeights);
+ continue;
+ }
+ case 'h': { // Basic block hash secifier.
+ if ( WI == ProgramWeightAndHashInfo.end())
+ continue;
+ BlockHashMap BlockHashes;
+ for (size_t I = 0; I < Values.size(); ++I) {
+ if (auto Err = parseBBHash(Values[I], BlockHashes))
+ return Err;
+ }
+ WI->second.BlockHashes = std::move(BlockHashes);
+ continue;
+ }
default:
return createProfileParseError(Twine("invalid specifier: '") +
Twine(Specifier) + "'");
@@ -439,6 +524,12 @@ BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction(
return BBSPR.getClonePathsForFunction(FuncName);
}
+std::pair<bool, WeightAndHashInfo>
+BasicBlockSectionsProfileReaderWrapperPass::getWeightAndHashInfoForFunction(
+ StringRef FuncName) const {
+ return BBSPR.getWeightAndHashInfoForFunction(FuncName);
+}
+
BasicBlockSectionsProfileReader &
BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() {
return BBSPR;
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 5dd6413431255..1614fa0d41ba4 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -35,6 +35,7 @@ add_llvm_component_library(LLVMCodeGen
BasicBlockSections.cpp
BasicBlockPathCloning.cpp
BasicBlockSectionsProfileReader.cpp
+ BasicBlockMatchingAndInference.cpp
CalcSpillWeights.cpp
CallBrPrepare.cpp
CallingConvLower.cpp
@@ -108,6 +109,7 @@ add_llvm_component_library(LLVMCodeGen
LowerEmuTLS.cpp
MachineBasicBlock.cpp
MachineBlockFrequencyInfo.cpp
+ MachineBlockHashInfo.cpp
MachineBlockPlacement.cpp
MachineBranchProbabilityInfo.cpp
MachineCFGPrinter.cpp
diff --git a/llvm/lib/CodeGen/MachineBlockHashInfo.cpp b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp
new file mode 100644
index 0000000000000..a9636bd09a1a2
--- /dev/null
+++ b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp
@@ -0,0 +1,111 @@
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+using OperandHashFuncTy =
+ function_ref<uint64_t(uint64_t &, const MachineOperand &)>;
+
+uint64_t hashBlock(const MachineBasicBlock &MBB,
+ OperandHashFuncTy OperandHashFunc) {
+ uint64_t Hash = 0;
+ for (const MachineInstr &MI : MBB) {
+ if (MI.isMetaInstruction())
+ continue;
+ // Ignore terminator instruction
+ if (MI.isTerminator())
+ continue;
+ Hash = hashing::detail::hash_16_bytes(Hash, MI.getOpcode());
+ for (unsigned i = 0; i < MI.getNumOperands(); i++) {
+ Hash = OperandHashFunc(Hash, MI.getOperand(i));
+ }
+ }
+ return Hash;
+}
+
+/// Hashing a 64-bit integer to a 16-bit one.
+uint16_t hash_64_to_16(const uint64_t Hash) {
+ uint16_t Res = (uint16_t)(Hash & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 16) & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 32) & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 48) & 0xFFFF);
+ return Res;
+}
+
+uint64_t hashInstOperand(uint64_t &Hash, const MachineOperand &Operand) {
+ return hashing::detail::hash_16_bytes(Hash, hash_value(Operand));
+}
+
+INITIALIZE_PASS(MachineBlockHashInfo, "machine-block-hash",
+ "Machine Block Hash Analysis", true, true)
+
+char MachineBlockHashInfo::ID = 0;
+
+MachineBlockHashInfo::MachineBlockHashInfo() : MachineFunctionPass(ID) {
+ initializeMachineBlockHashInfoPass(*PassRegistry::getPassRegistry());
+}
+
+void MachineBlockHashInfo::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool MachineBlockHashInfo::runOnMachineFunction(MachineFunction &F) {
+ DenseMap<MachineBasicBlock *, BlendedBlockHash> BlendedHashes;
+ DenseMap<MachineBasicBlock *, uint64_t> OpcodeHashes;
+ uint16_t Offset = 0;
+ // Initialize hash components
+ for (MachineBasicBlock &MBB : F) {
+ BlendedBlockHash BlendedHash;
+ // offset of the machine basic block
+ BlendedHash.Offset = Offset;
+ Offset += MBB.size();
+ // Hashing opcodes
+ uint64_t OpcodeHash = hashBlock(
+ MBB, [](uint64_t &Hash, const MachineOperand &Op) { return Hash; });
+ OpcodeHashes[&MBB] = OpcodeHash;
+ BlendedHash.OpcodeHash = hash_64_to_16(OpcodeHash);
+ // Hash complete instructions
+ uint64_t InstrHash = hashBlock(MBB, hashInstOperand);
+ BlendedHash.InstrHash = hash_64_to_16(InstrHash);
+ BlendedHashes[&MBB] = BlendedHash;
+ }
+
+ // Initialize neighbor hash
+ for (MachineBasicBlock &MBB : F) {
+ uint64_t Hash = OpcodeHashes[&MBB];
+ // Append hashes of successors
+ for (MachineBasicBlock *SuccMBB : MBB.successors()) {
+ uint64_t SuccHash = OpcodeHashes[SuccMBB];
+ Hash = hashing::detail::hash_16_bytes(Hash, SuccHash);
+ }
+ // Append hashes of predecessors
+ for (MachineBasicBlock *PredMBB : MBB.predecessors()) {
+ uint64_t PredHash = OpcodeHashes[PredMBB];
+ Hash = hashing::detail::hash_16_bytes(Hash, PredHash);
+ }
+ BlendedHashes[&MBB].NeighborHash = hash_64_to_16(Hash);
+ }
+
+ // Assign hashes
+ for (MachineBasicBlock &MBB : F) {
+ if (MBB.getBBID()) {
+ MBBHashInfo[MBB.getBBID()->BaseID] = BlendedHashes[&MBB].combine();
+ }
+ }
+
+ return false;
+}
+
+uint64_t MachineBlockHashInfo::getMBBHash(const MachineBasicBlock &MBB) {
+ if (MBB.getBBID()) {
+ return MBBHashInfo[MBB.getBBID()->BaseID];
+ }
+ return 0;
+}
+
+MachineFunctionPass *llvm::createMachineBlockHashInfoPass() {
+ return new MachineBlockHashInfo();
+}
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 0095ce3d96277..9c3d9c28d18eb 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -266,6 +266,16 @@ static cl::opt<bool>
cl::desc("Split static data sections into hot and cold "
"sections using profile information"));
+/// Enable matching and inference when using propeller.
+static cl::opt<bool>
+ PropellerMatchInfer("propeller-match-infer",
+ cl::desc("Use match&infer to evaluate stale profile"),
+ cl::init(false), cl::Optional);
+
+cl::opt<bool> EmitBBHash("emit-bb-hash",
+ cl::desc("Emit the hash of basic block in the SHT_LLVM_BB_ADDR_MAP section."),
+ cl::init(false), cl::Optional);
+
/// Allow standard passes to be disabled by command line options. This supports
/// simple binary flags that either suppress the pass or do nothing.
/// i.e. -disable-mypass=false has no effect.
@@ -1275,10 +1285,15 @@ void TargetPassConfig::addMachinePasses() {
// address map (or both).
if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
TM->Options.BBAddrMap) {
+ if (EmitBBHash || PropellerMatchInfer)
+ addPass(llvm::createMachineBlockHashInfoPass());
if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(
TM->getBBSectionsFuncListBuf()));
- addPass(llvm::createBasicBlockPathCloningPass());
+ if (PropellerMatchInfer)
+ addPass(llvm::createBasicBlockMatchingAndInferencePass());
+ else
+ addPass(llvm::createBasicBlockPathCloningPass());
}
addPass(llvm::createBasicBlockSectionsPass());
}
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index bf42c92a242a1..c1faf6a401345 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -873,6 +873,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
uint32_t Offset = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
uint32_t MD = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
+ uint64_t Hash = FeatEnable.BBHash ? readULEB128As<uint64_t>(Data, Cur, ULEBSizeErr) : 0;
if (Version >= 1) {
// Offset is calculated relative to the end of the previous BB.
Offset += PrevBBEndOffset;
@@ -884,7 +885,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
MetadataDecodeErr = MetadataOrErr.takeError();
break;
}
- BBEntries.push_back({ID, Offset, Size, *MetadataOrErr});
+ BBEntries.push_back({ID, Offset, Size, *MetadataOrErr, Hash});
}
TotalNumBlocks += BBEntries.size();
}
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 9ae76a71ede5e..02e41c886f34d 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -1502,6 +1502,9 @@ void ELFState<ELFT>::writeSectionContent(
SHeader.sh_size += CBA.writeULEB128(BBE.AddressOffset);
SHeader.sh_size += CBA.writeULEB128(BBE.Size);
SHeader.sh_size += CBA.writeULEB128(BBE.Metadata);
+ if (FeatureOrErr->BBHash) {
+ SHeader.sh_size += CBA.writeULEB128(BBE.Hash);
+ }
}
}
if (!PGOAnalyses)
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 1f970739c1e7e..35001aca036ec 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -1872,6 +1872,7 @@ void MappingTraits<ELFYAML::BBAddrMapEntry::BBEntry>::mapping(
IO.mapRequired("AddressOffset", E.AddressOffset);
IO.mapRequired("Size", E.Size);
IO.mapRequired("Metadata", E.Metadata);
+ IO.mapOptional("Hash", E.Hash);
}
void MappingTraits<ELFYAML::PGOAnalysisMapEntry>::mapping(
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll
new file mode 100644
index 0000000000000..809621197dddf
--- /dev/null
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-bb-hash.ll
@@ -0,0 +1,86 @@
+; Check the basic block sections labels option when emit basic block hash.
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -emit-bb-hash | FileCheck %s --check-prefixes=CHECK,UNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-address-map -emit-bb-hash | FileCheck %s --check-prefixes=CHECK,NOUNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -split-machine-functions -emit-bb-hash | FileCheck %s --check-prefixes=CHECK,UNIQ
+
+define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 {
+ br i1 %0, label %3, label %8
+
+3:
+ %4 = invoke i32 @_Z3barv()
+ to label %8 unwind label %6
+ br label %10
+
+6:
+ landingpad { ptr, i32 }
+ catch ptr null
+ br label %12
+
+8:
+ %9 = call i32 @_Z3foov()
+ br i1 %1, label %12, label %10
+
+10:
+ %11 = select i1 %1, ptr blockaddress(@_Z3bazb, %3), ptr blockaddress(@_Z3bazb, %12) ; <ptr> [#uses=1]
+ indirectbr ptr %11, [label %3, label %12]
+
+12:
+ ret void
+}
+
+declare i32 @_Z3barv() #1
+
+declare i32 @_Z3foov() #1
+
+declare i32 @__gxx_personality_v0(...)
+
+; UNIQ: .section .text._Z3bazb,"ax", at progbits{{$}}
+; NOUNIQ: .section .text,"ax", at progbits,unique,1
+; CHECK-LABEL: _Z3bazb:
+; CHECK-LABEL: .Lfunc_begin0:
+; CHECK-LABEL: .LBB_END0_0:
+; CHECK-LABEL: .LBB0_1:
+; CHECK-LABEL: .LBB_END0_1:
+; CHECK-LABEL: .LBB0_2:
+; CHECK-LABEL: .LBB_END0_2:
+; CHECK-LABEL: .LBB0_3:
+; CHECK-LABEL: .LBB_END0_3:
+; CHECK-LABEL: .Lfunc_end0:
+
+; UNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3bazb{{$}}
+;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
+; NOUNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text,unique,1
+; CHECK-NEXT: .byte 2 # version
+; CHECK-NEXT: .byte 32 # feature
+; CHECK-NEXT: .quad .Lfunc_begin0 # function address
+; CHECK-NEXT: .byte 6 # number of basic blocks
+; CHECK-NEXT: .byte 0 # BB id
+; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0
+; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0
+; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 1 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_1-.LBB_END0_0
+; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1
+; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 3 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_2-.LBB_END0_1
+; CHECK-NEXT: .uleb128 .LBB_END0_2-.LBB0_2
+; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 4 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_3-.LBB_END0_2
+; CHECK-NEXT: .uleb128 .LBB_END0_3-.LBB0_3
+; CHECK-NEXT: .byte 16
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 5 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_4-.LBB_END0_3
+; CHECK-NEXT: .uleb128 .LBB_END0_4-.LBB0_4
+; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .ascii "{{.*}}"
+; CHECK-NEXT: .byte 2 # BB id
+; CHECK-NEXT: .uleb128 .LBB0_5-.LBB_END0_4
+; CHECK-NEXT: .uleb128 .LBB_END0_5-.LBB0_5
+; CHECK-NEXT: .byte 5
+; CHECK-NEXT: .ascii "{{.*}}"
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll
new file mode 100644
index 0000000000000..8ac034f25940d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters-with-match-infer.ll
@@ -0,0 +1,90 @@
+; BB cluster section with -propeller-match-infer tests.
+;
+; Test1: Basic blocks #0 (entry), #1, and #3 will be placed in the same section.
+; The rest will be placed in the cold section.
+; RUN: echo 'v1' > %t1
+; RUN: echo 'f foo' >> %t1
+; RUN: echo 'c 0 1 3' >> %t1
+; RUN: echo 'w 0:100,1:100,2:0 1:100,3:100 2:0,3:0 3:100' >> %t1
+; RUN: echo 'h 0:10838971302264569856 1:10005270435075325958 2:10005130895882846216 3:17705462075698380809' >> %t1
+;
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -propeller-match-infer -basic-block-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1
+;
+; Test2: Basic blocks #0 (entry), #2, and #3 will be placed in the same section.
+; The rest will be placed in the cold section.
+; RUN: echo 'v1' > %t2
+; RUN: echo 'f foo' >> %t2
+; RUN: echo 'c 0 2 3' >> %t2
+; RUN: echo 'w 0:100,1:0,2:100 1:0,3:0 2:100,3:100 3:100' >> %t2
+; RUN: echo 'h 0:10838971302264569856 1:10005270435075325958 2:10005130895882846216 3:17705462075698380809' >> %t2
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -propeller-match-infer -basic-block-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2
+
+define void @foo(i1 zeroext) nounwind {
+ %2 = alloca i8, align 1
+ %3 = zext i1 %0 to i8
+ store i8 %3, ptr %2, align 1
+ %4 = load i8, ptr %2, align 1
+ %5 = trunc i8 %4 to i1
+ br i1 %5, label %6, label %8
+
+6: ; preds = %1
+ %7 = call i32 @bar()
+ br label %10
+
+8: ; preds = %1
+ %9 = call i32 @baz()
+ br label %10
+
+10: ; preds = %8, %6
+ ret void
+}
+
+declare i32 @bar() #1
+
+declare i32 @baz() #1
+
+; LINUX-SECTIONS1: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-LABEL: foo:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: # %bb.0:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: # %bb.1:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: .LBB0_3:
+; LINUX-SECTIONS1-NOT: .section
+; LINUX-SECTIONS1-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS1-LABEL: .LBB_END0_3:
+; LINUX-SECTIONS1: .section .text.split.foo,"ax", at progbits
+; LINUX-SECTIONS1-LABEL: foo.cold:
+; LINUX-SECTIONS1-LABEL: .LBB_END0_2:
+; LINUX-SECTIONS1-NEXT: .size foo.cold, .LBB_END0_2-foo.cold
+; LINUX-SECTIONS1: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS1-LABEL: .Lfunc_end0:
+; LINUX-SECTIONS1-NEXT: .size foo, .Lfunc_end0-foo
+
+; LINUX-SECTIONS2: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-LABEL: foo:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: # %bb.0:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: # %bb.2:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: .LBB0_3:
+; LINUX-SECTIONS2-NOT: .section
+; LINUX-SECTIONS2-NOT: .LBB_END0_{{[0-9]+}}
+; LINUX-SECTIONS2-LABEL: .LBB_END0_3:
+; LINUX-SECTIONS2: .section .text.split.foo,"ax", at progbits
+; LINUX-SECTIONS2-LABEL: foo.cold:
+; LINUX-SECTIONS2-LABEL: .LBB_END0_1:
+; LINUX-SECTIONS2-NEXT: .size foo.cold, .LBB_END0_1-foo.cold
+; LINUX-SECTIONS2: .section .text.foo,"ax", at progbits
+; LINUX-SECTIONS2-LABEL: .Lfunc_end0:
+; LINUX-SECTIONS2-NEXT: .size foo, .Lfunc_end0-foo
diff --git a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
index 299bf463cf4bc..93cf7da40c9c0 100644
--- a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
@@ -23,14 +23,17 @@
# VALID-NEXT: AddressOffset: 0x1
# VALID-NEXT: Size: 0x2
# VALID-NEXT: Metadata: 0x3
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 2
# VALID-NEXT: AddressOffset: 0x4
# VALID-NEXT: Size: 0x5
# VALID-NEXT: Metadata: 0x6
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 4
# VALID-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
# VALID-NEXT: Size: 0xFFFFFFFFFFFFFFF8
# VALID-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - Version: 2
# VALID-NEXT: Feature: 0xA
# VALID-NEXT: BBRanges:
@@ -40,6 +43,7 @@
# VALID-NEXT: AddressOffset: 0xA
# VALID-NEXT: Size: 0xB
# VALID-NEXT: Metadata: 0xC
+# VALID-NEXT: Hash: 0
# VALID-NEXT: PGOAnalyses:
# VALID-NEXT: - FuncEntryCount: 100
# VALID-NEXT: PGOBBEntries:
@@ -135,6 +139,7 @@ Sections:
# MULTI-NEXT: AddressOffset: 0x1
# MULTI-NEXT: Size: 0x2
# MULTI-NEXT: Metadata: 0x3
+# MULTI-NEXT: Hash: 0
# MULTI-NEXT: PGOAnalyses:
# MULTI-NEXT: - FuncEntryCount: 0
# MULTI-NEXT: PGOBBEntries:
diff --git a/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml b/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
index 8dbf97ef2bc12..aed4beff1d47c 100644
--- a/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
@@ -22,14 +22,17 @@
# VALID-NEXT: AddressOffset: 0x1
# VALID-NEXT: Size: 0x2
# VALID-NEXT: Metadata: 0x3
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 2
# VALID-NEXT: AddressOffset: 0x4
# VALID-NEXT: Size: 0x5
# VALID-NEXT: Metadata: 0x6
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - ID: 4
# VALID-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
# VALID-NEXT: Size: 0xFFFFFFFFFFFFFFF8
# VALID-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
+# VALID-NEXT: Hash: 0
# VALID-NEXT: - Version: 2
# VALID-NEXT: Feature: 0x8
# VALID-NEXT: BBRanges:
@@ -39,6 +42,7 @@
# VALID-NEXT: AddressOffset: 0xA
# VALID-NEXT: Size: 0xB
# VALID-NEXT: Metadata: 0xC
+# VALID-NEXT: Hash: 0
--- !ELF
FileHeader:
@@ -126,6 +130,7 @@ Sections:
# MULTI-NEXT: AddressOffset: 0x1
# MULTI-NEXT: Size: 0x2
# MULTI-NEXT: Metadata: 0x3
+# MULTI-NEXT: Hash: 0
# MULTI-NEXT: - Name: '.llvm_bb_addr_map (1)'
# MULTI-NEXT: Type: SHT_LLVM_BB_ADDR_MAP
# MULTI-NEXT: Entries:
diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp
index 493e673d6a07d..91ecec891458d 100644
--- a/llvm/unittests/Object/ELFObjectFileTest.cpp
+++ b/llvm/unittests/Object/ELFObjectFileTest.cpp
@@ -977,6 +977,124 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
}
}
+// Test for the ELFObjectFile::readBBAddrMap API with BBHash.
+TEST(ELFObjectFileTest, ReadBBHash) {
+ StringRef CommonYamlString(R"(
+--- !ELF
+FileHeader:
+ Class: ELFCLASS64
+ Data: ELFDATA2LSB
+ Type: ET_EXEC
+Sections:
+ - Name: .llvm_bb_addr_map_1
+ Type: SHT_LLVM_BB_ADDR_MAP
+ Link: 1
+ Entries:
+ - Version: 2
+ Feature: 0x20
+ BBRanges:
+ - BaseAddress: 0x11111
+ BBEntries:
+ - ID: 1
+ AddressOffset: 0x0
+ Size: 0x1
+ Metadata: 0x2
+ Hash: 0x1
+ - Name: .llvm_bb_addr_map_2
+ Type: SHT_LLVM_BB_ADDR_MAP
+ Link: 1
+ Entries:
+ - Version: 2
+ Feature: 0x28
+ BBRanges:
+ - BaseAddress: 0x22222
+ BBEntries:
+ - ID: 2
+ AddressOffset: 0x0
+ Size: 0x2
+ Metadata: 0x4
+ Hash: 0x2
+ - BaseAddress: 0xFFFFF
+ BBEntries:
+ - ID: 15
+ AddressOffset: 0xF0
+ Size: 0xF1
+ Metadata: 0x1F
+ Hash: 0xF
+ - Name: .llvm_bb_addr_map_3
+ Type: SHT_LLVM_BB_ADDR_MAP
+ Link: 2
+ Entries:
+ - Version: 2
+ Feature: 0x20
+ BBRanges:
+ - BaseAddress: 0x33333
+ BBEntries:
+ - ID: 0
+ AddressOffset: 0x0
+ Size: 0x3
+ Metadata: 0x6
+ Hash: 0x3
+ - Name: .llvm_bb_addr_map_4
+ Type: SHT_LLVM_BB_ADDR_MAP
+ # Link: 0 (by default, can be overriden)
+ Entries:
+ - Version: 2
+ Feature: 0x20
+ BBRanges:
+ - BaseAddress: 0x44444
+ BBEntries:
+ - ID: 0
+ AddressOffset: 0x0
+ Size: 0x4
+ Metadata: 0x18
+ Hash: 0x4
+)");
+
+ BBAddrMap E1 = {
+ {{0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}, 0x1}}}}};
+ BBAddrMap E2 = {
+ {{0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}, 0x2}}},
+ {0xFFFFF, {{15, 0xF0, 0xF1, {true, true, true, true, true}, 0xF}}}}};
+ BBAddrMap E3 = {
+ {{0x33333, {{0, 0x0, 0x3, {false, true, true, false, false}, 0x3}}}}};
+ BBAddrMap E4 = {
+ {{0x44444, {{0, 0x0, 0x4, {false, false, false, true, true}, 0x4}}}}};
+
+ std::vector<BBAddrMap> Section0BBAddrMaps = {E4};
+ std::vector<BBAddrMap> Section1BBAddrMaps = {E3};
+ std::vector<BBAddrMap> Section2BBAddrMaps = {E1, E2};
+ std::vector<BBAddrMap> AllBBAddrMaps = {E1, E2, E3, E4};
+
+ auto DoCheckSucceeds = [&](StringRef YamlString,
+ std::optional<unsigned> TextSectionIndex,
+ std::vector<BBAddrMap> ExpectedResult) {
+ SCOPED_TRACE("for TextSectionIndex: " +
+ (TextSectionIndex ? llvm::Twine(*TextSectionIndex) : "{}") +
+ " and object yaml:\n" + YamlString);
+ SmallString<0> Storage;
+ Expected<ELFObjectFile<ELF64LE>> ElfOrErr =
+ toBinary<ELF64LE>(Storage, YamlString);
+ ASSERT_THAT_EXPECTED(ElfOrErr, Succeeded());
+
+ Expected<const typename ELF64LE::Shdr *> BBAddrMapSecOrErr =
+ ElfOrErr->getELFFile().getSection(1);
+ ASSERT_THAT_EXPECTED(BBAddrMapSecOrErr, Succeeded());
+ auto BBAddrMaps = ElfOrErr->readBBAddrMap(TextSectionIndex);
+ ASSERT_THAT_EXPECTED(BBAddrMaps, Succeeded());
+ EXPECT_EQ(*BBAddrMaps, ExpectedResult);
+ };
+
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/std::nullopt,
+ AllBBAddrMaps);
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/0,
+ Section0BBAddrMaps);
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/2,
+ Section1BBAddrMaps);
+ DoCheckSucceeds(CommonYamlString, /*TextSectionIndex=*/1,
+ Section2BBAddrMaps);
+}
+
// Test for the ELFObjectFile::readBBAddrMap API with PGOAnalysisMap.
TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
StringRef CommonYamlString(R"(
diff --git a/llvm/unittests/Object/ELFTypesTest.cpp b/llvm/unittests/Object/ELFTypesTest.cpp
index 13130dde80ef1..44cd8d397717a 100644
--- a/llvm/unittests/Object/ELFTypesTest.cpp
+++ b/llvm/unittests/Object/ELFTypesTest.cpp
@@ -125,9 +125,9 @@ TEST(ELFTypesTest, BBAddrMapFeaturesEncodingTest) {
TEST(ELFTypesTest, BBAddrMapFeaturesInvalidEncodingTest) {
const std::array<std::string, 2> Errors = {
- "invalid encoding for BBAddrMap::Features: 0x20",
+ "invalid encoding for BBAddrMap::Features: 0x40",
"invalid encoding for BBAddrMap::Features: 0xf0"};
- const std::array<uint8_t, 2> Values = {{0b10'0000, 0b1111'0000}};
+ const std::array<uint8_t, 2> Values = {{0b100'0000, 0b1111'0000}};
for (const auto &[Val, Error] : llvm::zip(Values, Errors)) {
EXPECT_THAT_ERROR(BBAddrMap::Features::decode(Val).takeError(),
FailedWithMessage(Error));
More information about the llvm-commits
mailing list