[clang] [lld] [llvm] Adding Matching and Inference Functionality to Propeller (PR #139008)
via llvm-commits
llvm-commits at lists.llvm.org
Wed May 7 20:13:04 PDT 2025
https://github.com/wdx727 created https://github.com/llvm/llvm-project/pull/139008
We would like to add the matching and inference functionality to Propeller to address the issue that Propeller has zero tolerance for changes in source code and compilation parameters. The complete RFC can be found at the following URL: https://discourse.llvm.org/t/rfc-adding-matching-and-inference-functionality-to-propeller/86238/1.
>From e04f4df5c1c1425804b41c62a8a18d3f267eaadd Mon Sep 17 00:00:00 2001
From: wudexin <wudexin at kuaishou.com>
Date: Mon, 14 Apr 2025 21:07:58 +0800
Subject: [PATCH 1/2] Add a new MachineBlockHashInfo pass to calculate the hash
value of Machine Basic Blocks and write the hash values into the
bb_address_map section.
---
.../llvm/CodeGen/MachineBlockHashInfo.h | 108 ++++++++++++++++++
llvm/include/llvm/InitializePasses.h | 1 +
llvm/include/llvm/Object/ELFTypes.h | 5 +-
llvm/include/llvm/ObjectYAML/ELFYAML.h | 1 +
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 6 +
llvm/lib/CodeGen/BasicBlockSections.cpp | 3 +
llvm/lib/CodeGen/CMakeLists.txt | 1 +
llvm/lib/CodeGen/CodeGen.cpp | 1 +
llvm/lib/CodeGen/MachineBlockHashInfo.cpp | 104 +++++++++++++++++
llvm/lib/Object/ELF.cpp | 3 +-
llvm/lib/ObjectYAML/ELFEmitter.cpp | 1 +
llvm/lib/ObjectYAML/ELFYAML.cpp | 1 +
llvm/test/CodeGen/AArch64/O0-pipeline.ll | 1 +
llvm/test/CodeGen/AArch64/O3-pipeline.ll | 1 +
.../AArch64/arm64-opt-remarks-lazy-bfi.ll | 2 +
llvm/test/CodeGen/X86/O0-pipeline.ll | 1 +
.../basic-block-address-map-pgo-features.ll | 6 +
...k-address-map-with-basic-block-sections.ll | 5 +-
.../X86/basic-block-address-map-with-mfs.ll | 3 +
.../CodeGen/X86/basic-block-address-map.ll | 8 +-
llvm/test/CodeGen/X86/opt-pipeline.ll | 1 +
...ddrmap-disassemble-symbolize-operands.yaml | 18 +++
.../elf-bbaddrmap-symbolize-relocatable.yaml | 10 +-
.../llvm-objdump/X86/elf-pgoanalysismap.yaml | 9 ++
.../ELF/bb-addr-map-pgo-analysis-map.test | 16 ++-
.../ELF/bb-addr-map-relocatable.test | 11 +-
.../tools/llvm-readobj/ELF/bb-addr-map.test | 14 ++-
.../ELF/bb-addr-map-pgo-analysis-map.yaml | 10 ++
llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml | 10 ++
.../ELF/bb-addr-map-pgo-analysis-map.yaml | 9 +-
llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml | 9 +-
llvm/tools/llvm-readobj/ELFDumper.cpp | 1 +
llvm/tools/obj2yaml/elf2yaml.cpp | 3 +-
llvm/unittests/Object/ELFObjectFileTest.cpp | 87 +++++++++-----
34 files changed, 420 insertions(+), 50 deletions(-)
create mode 100644 llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
create mode 100644 llvm/lib/CodeGen/MachineBlockHashInfo.cpp
diff --git a/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
new file mode 100644
index 0000000000000..a943f71357e6e
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h
@@ -0,0 +1,108 @@
+#ifndef LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
+#define LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+namespace llvm {
+
+/// An object wrapping several components of a basic block hash. The combined
+/// (blended) hash is represented and stored as one uint64_t, while individual
+/// components are of smaller size (e.g., uint16_t or uint8_t).
+struct BlendedBlockHash {
+private:
+ static uint64_t combineHashes(uint16_t Hash1, uint16_t Hash2, uint16_t Hash3,
+ uint16_t Hash4) {
+ uint64_t Hash = 0;
+
+ Hash |= uint64_t(Hash4);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash3);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash2);
+ Hash <<= 16;
+
+ Hash |= uint64_t(Hash1);
+
+ return Hash;
+ }
+
+ static void parseHashes(uint64_t Hash, uint16_t &Hash1, uint16_t &Hash2,
+ uint16_t &Hash3, uint16_t &Hash4) {
+ Hash1 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash2 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash3 = Hash & 0xffff;
+ Hash >>= 16;
+
+ Hash4 = Hash & 0xffff;
+ Hash >>= 16;
+ }
+
+public:
+ explicit BlendedBlockHash() {}
+
+ explicit BlendedBlockHash(uint64_t CombinedHash) {
+ parseHashes(CombinedHash, Offset, OpcodeHash, InstrHash, NeighborHash);
+ }
+
+ /// Combine the blended hash into uint64_t.
+ uint64_t combine() const {
+ return combineHashes(Offset, OpcodeHash, InstrHash, NeighborHash);
+ }
+
+ /// Compute a distance between two given blended hashes. The smaller the
+ /// distance, the more similar two blocks are. For identical basic blocks,
+ /// the distance is zero.
+ uint64_t distance(const BlendedBlockHash &BBH) const {
+ assert(OpcodeHash == BBH.OpcodeHash &&
+ "incorrect blended hash distance computation");
+ uint64_t Dist = 0;
+ // Account for NeighborHash
+ Dist += NeighborHash == BBH.NeighborHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for InstrHash
+ Dist += InstrHash == BBH.InstrHash ? 0 : 1;
+ Dist <<= 16;
+ // Account for Offset
+ Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset);
+ return Dist;
+ }
+
+ /// The offset of the basic block from the function start.
+ uint16_t Offset{0};
+ /// (Loose) Hash of the basic block instructions, excluding operands.
+ uint16_t OpcodeHash{0};
+ /// (Strong) Hash of the basic block instructions, including opcodes and
+ /// operands.
+ uint16_t InstrHash{0};
+ /// Hash of the (loose) basic block together with (loose) hashes of its
+ /// successors and predecessors.
+ uint16_t NeighborHash{0};
+};
+
+class MachineBlockHashInfo : public MachineFunctionPass {
+ DenseMap<unsigned, uint64_t> MBBHashInfo;
+
+public:
+ static char ID;
+ MachineBlockHashInfo();
+
+ StringRef getPassName() const override {
+ return "Basic Block Hash Compute";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+
+ uint64_t getMBBHash(const MachineBasicBlock &MBB);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H
\ No newline at end of file
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 9fd5e7676b19d..38289da1449d5 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -184,6 +184,7 @@ void initializeMIRCanonicalizerPass(PassRegistry &);
void initializeMIRNamerPass(PassRegistry &);
void initializeMIRPrintingPassPass(PassRegistry &);
void initializeMachineBlockFrequencyInfoWrapperPassPass(PassRegistry &);
+void initializeMachineBlockHashInfoPass(PassRegistry&);
void initializeMachineBlockPlacementLegacyPass(PassRegistry &);
void initializeMachineBlockPlacementStatsPass(PassRegistry &);
void initializeMachineBranchProbabilityInfoWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index 87e4dbe448091..40b7ec378a334 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -914,9 +914,10 @@ struct BBAddrMap {
uint32_t Size = 0; // Size of the basic block.
Metadata MD = {false, false, false, false,
false}; // Metdata for this basic block.
+ uint64_t Hash = 0;
- BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD)
- : ID(ID), Offset(Offset), Size(Size), MD(MD){};
+ BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD, uint64_t Hash)
+ : ID(ID), Offset(Offset), Size(Size), MD(MD), Hash(Hash){};
bool operator==(const BBEntry &Other) const {
return ID == Other.ID && Offset == Other.Offset && Size == Other.Size &&
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index dfdfa055d65fa..9427042db4303 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -162,6 +162,7 @@ struct BBAddrMapEntry {
llvm::yaml::Hex64 AddressOffset;
llvm::yaml::Hex64 Size;
llvm::yaml::Hex64 Metadata;
+ llvm::yaml::Hex64 Hash;
};
uint8_t Version;
llvm::yaml::Hex8 Feature;
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index cf8f1c878ea5a..cf38279f0d770 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -40,6 +40,7 @@
#include "llvm/CodeGen/GCMetadataPrinter.h"
#include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineDominators.h"
@@ -450,6 +451,7 @@ const MCSection *AsmPrinter::getCurrentSection() const {
void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
MachineFunctionPass::getAnalysisUsage(AU);
+ AU.addRequired<MachineBlockHashInfo>();
AU.addRequired<MachineOptimizationRemarkEmitterPass>();
AU.addRequired<GCModuleInfo>();
AU.addRequired<LazyMachineBlockFrequencyInfoPass>();
@@ -1477,6 +1479,8 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
PrevMBBEndSymbol = MBBSymbol;
}
+ auto MBHI = &getAnalysis<MachineBlockHashInfo>();
+
if (!Features.OmitBBEntries) {
// TODO: Remove this check when version 1 is deprecated.
if (BBAddrMapVersion > 1) {
@@ -1496,6 +1500,8 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
emitLabelDifferenceAsULEB128(MBB.getEndSymbol(), MBBSymbol);
// Emit the Metadata.
OutStreamer->emitULEB128IntValue(getBBAddrMapMetadata(MBB));
+ // Emit the Hash.
+ OutStreamer->emitULEB128IntValue(MBHI->getMBBHash(MBB));
}
PrevMBBEndSymbol = MBB.getEndSymbol();
diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp
index 1eedfc4b25912..b02a7afe771cd 100644
--- a/llvm/lib/CodeGen/BasicBlockSections.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSections.cpp
@@ -70,6 +70,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
#include "llvm/CodeGen/MachineDominators.h"
@@ -134,6 +135,7 @@ INITIALIZE_PASS_BEGIN(
"Prepares for basic block sections, by splitting functions "
"into clusters of basic blocks.",
false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockHashInfo)
INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)
INITIALIZE_PASS_END(BasicBlockSections, "bbsections-prepare",
"Prepares for basic block sections, by splitting functions "
@@ -400,6 +402,7 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
+ AU.addRequired<MachineBlockHashInfo>();
AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
AU.addUsedIfAvailable<MachineDominatorTreeWrapperPass>();
AU.addUsedIfAvailable<MachinePostDominatorTreeWrapperPass>();
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 5dd6413431255..424f7b98e2c91 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -108,6 +108,7 @@ add_llvm_component_library(LLVMCodeGen
LowerEmuTLS.cpp
MachineBasicBlock.cpp
MachineBlockFrequencyInfo.cpp
+ MachineBlockHashInfo.cpp
MachineBlockPlacement.cpp
MachineBranchProbabilityInfo.cpp
MachineCFGPrinter.cpp
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 8b777ed2bbc9b..0ff5d6006c87a 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -72,6 +72,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
initializeMIRNamerPass(Registry);
initializeMIRProfileLoaderPassPass(Registry);
initializeMachineBlockFrequencyInfoWrapperPassPass(Registry);
+ initializeMachineBlockHashInfoPass(Registry);
initializeMachineBlockPlacementLegacyPass(Registry);
initializeMachineBlockPlacementStatsPass(Registry);
initializeMachineCFGPrinterPass(Registry);
diff --git a/llvm/lib/CodeGen/MachineBlockHashInfo.cpp b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp
new file mode 100644
index 0000000000000..1bf5466fae9b3
--- /dev/null
+++ b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp
@@ -0,0 +1,104 @@
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+using OperandHashFuncTy = function_ref<uint64_t(uint64_t &, const MachineOperand &)>;
+
+uint64_t hashBlock(const MachineBasicBlock &MBB, OperandHashFuncTy OperandHashFunc) {
+ uint64_t Hash = 0;
+ for (const MachineInstr &MI : MBB) {
+ if (MI.isPseudo())
+ continue;
+ // Ignore unconditional jumps
+ if (MI.isUnconditionalBranch())
+ continue;
+ Hash = hashing::detail::hash_16_bytes(Hash, MI.getOpcode());
+ for (unsigned i = 0; i < MI.getNumOperands(); i++) {
+ Hash = OperandHashFunc(Hash, MI.getOperand(i));
+ }
+ }
+ return Hash;
+}
+
+/// Hashing a 64-bit integer to a 16-bit one.
+uint16_t hash_64_to_16(const uint64_t Hash) {
+ uint16_t Res = (uint16_t)(Hash & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 16) & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 32) & 0xFFFF);
+ Res ^= (uint16_t)((Hash >> 48) & 0xFFFF);
+ return Res;
+}
+
+uint64_t hashInstOperand(uint64_t &Hash, const MachineOperand &Operand) {
+ return hashing::detail::hash_16_bytes(Hash, hash_value(Operand));
+}
+
+INITIALIZE_PASS(MachineBlockHashInfo, "machine-block-hash",
+ "Machine Block Hash Analysis", true, true)
+
+char MachineBlockHashInfo::ID = 0;
+
+MachineBlockHashInfo::MachineBlockHashInfo() : MachineFunctionPass(ID) {
+ initializeMachineBlockHashInfoPass(*PassRegistry::getPassRegistry());
+}
+
+void MachineBlockHashInfo::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool MachineBlockHashInfo::runOnMachineFunction(MachineFunction &F) {
+ DenseMap<MachineBasicBlock *, BlendedBlockHash> BlendedHashes;
+ DenseMap<MachineBasicBlock *, uint64_t> OpcodeHashes;
+ uint16_t Offset = 0;
+ // Initialize hash components
+ for (MachineBasicBlock &MBB : F) {
+ BlendedBlockHash BlendedHash;
+ // offset of the machine basic block
+ BlendedHash.Offset = Offset;
+ Offset += MBB.size();
+ // Hashing opcodes
+ uint64_t OpcodeHash = hashBlock(MBB, [](uint64_t &Hash, const MachineOperand &Op) { return Hash; });
+ OpcodeHashes[&MBB] = OpcodeHash;
+ BlendedHash.OpcodeHash = hash_64_to_16(OpcodeHash);
+ // Hash complete instructions
+ uint64_t InstrHash = hashBlock(MBB, hashInstOperand);
+ BlendedHash.InstrHash = hash_64_to_16(InstrHash);
+ BlendedHashes[&MBB] = BlendedHash;
+ }
+
+ // Initialize neighbor hash
+ for (MachineBasicBlock &MBB : F) {
+ uint64_t Hash = OpcodeHashes[&MBB];
+ // Append hashes of successors
+ for (MachineBasicBlock *SuccMBB : MBB.successors()) {
+ uint64_t SuccHash = OpcodeHashes[SuccMBB];
+ Hash = hashing::detail::hash_16_bytes(Hash, SuccHash);
+ }
+ // Append hashes of predecessors
+ for (MachineBasicBlock *PredMBB : MBB.predecessors()) {
+ uint64_t PredHash = OpcodeHashes[PredMBB];
+ Hash = hashing::detail::hash_16_bytes(Hash, PredHash);
+ }
+ BlendedHashes[&MBB].NeighborHash = hash_64_to_16(Hash);
+ }
+
+ // Assign hashes
+ for (MachineBasicBlock &MBB : F) {
+ if (MBB.getBBID()) {
+ MBBHashInfo[MBB.getBBID()->BaseID] = BlendedHashes[&MBB].combine();
+ }
+ }
+
+ return false;
+}
+
+uint64_t MachineBlockHashInfo::getMBBHash(const MachineBasicBlock &MBB) {
+ if (MBB.getBBID()) {
+ return MBBHashInfo[MBB.getBBID()->BaseID];
+ }
+ return 0;
+}
\ No newline at end of file
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index bf42c92a242a1..dd585011a13ae 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -873,6 +873,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
uint32_t Offset = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
uint32_t MD = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
+ uint64_t Hash = readULEB128As<uint64_t>(Data, Cur, ULEBSizeErr);
if (Version >= 1) {
// Offset is calculated relative to the end of the previous BB.
Offset += PrevBBEndOffset;
@@ -884,7 +885,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
MetadataDecodeErr = MetadataOrErr.takeError();
break;
}
- BBEntries.push_back({ID, Offset, Size, *MetadataOrErr});
+ BBEntries.push_back({ID, Offset, Size, *MetadataOrErr, Hash});
}
TotalNumBlocks += BBEntries.size();
}
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 9ae76a71ede5e..a2542b63479ee 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -1502,6 +1502,7 @@ void ELFState<ELFT>::writeSectionContent(
SHeader.sh_size += CBA.writeULEB128(BBE.AddressOffset);
SHeader.sh_size += CBA.writeULEB128(BBE.Size);
SHeader.sh_size += CBA.writeULEB128(BBE.Metadata);
+ SHeader.sh_size += CBA.writeULEB128(BBE.Hash);
}
}
if (!PGOAnalyses)
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 1f970739c1e7e..ddbb47025f3d7 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -1872,6 +1872,7 @@ void MappingTraits<ELFYAML::BBAddrMapEntry::BBEntry>::mapping(
IO.mapRequired("AddressOffset", E.AddressOffset);
IO.mapRequired("Size", E.Size);
IO.mapRequired("Metadata", E.Metadata);
+ IO.mapRequired("Hash", E.Hash);
}
void MappingTraits<ELFYAML::PGOAnalysisMapEntry>::mapping(
diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll
index abc67eec32391..bf0abb42f386b 100644
--- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll
@@ -85,6 +85,7 @@
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: Stack Frame Layout Analysis
; CHECK-NEXT: Unpack machine instruction bundles
+; CHECK-NEXT: Basic Block Hash Compute
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: AArch64 Assembly Printer
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index e1481667a4ab7..ff4d4420de139 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -245,6 +245,7 @@
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: Stack Frame Layout Analysis
; CHECK-NEXT: Unpack machine instruction bundles
+; CHECK-NEXT: Basic Block Hash Compute
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: AArch64 Assembly Printer
diff --git a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll
index 08c314e538734..106ef26595eb5 100644
--- a/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-opt-remarks-lazy-bfi.ll
@@ -62,6 +62,7 @@
; HOTNESS-NEXT: Freeing Pass 'Unpack machine instruction bundles'
; HOTNESS-NEXT: Executing Pass 'Verify generated machine code'
; HOTNESS-NEXT: Freeing Pass 'Verify generated machine code'
+; HOTNESS-NEXT: Executing Pass 'Basic Block Hash Compute'
; HOTNESS-NEXT: Executing Pass 'Lazy Machine Block Frequency Analysis'
; HOTNESS-NEXT: Executing Pass 'Machine Optimization Remark Emitter'
; HOTNESS-NEXT: Building MachineBlockFrequencyInfo on the fly
@@ -107,6 +108,7 @@
; NO_HOTNESS-NEXT: Freeing Pass 'Unpack machine instruction bundles'
; NO_HOTNESS-NEXT: Executing Pass 'Verify generated machine code'
; NO_HOTNESS-NEXT: Freeing Pass 'Verify generated machine code'
+; NO_HOTNESS-NEXT: Executing Pass 'Basic Block Hash Compute'
; NO_HOTNESS-NEXT: Executing Pass 'Lazy Machine Block Frequency Analysis'
; NO_HOTNESS-NEXT: Executing Pass 'Machine Optimization Remark Emitter'
; NO_HOTNESS-NEXT: Executing Pass 'AArch64 Assembly Printer'
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index 6d824f8b510af..d2c286d13ff38 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -85,6 +85,7 @@
; CHECK-NEXT: X86 Load Value Injection (LVI) Ret-Hardening
; CHECK-NEXT: Pseudo Probe Inserter
; CHECK-NEXT: Unpack machine instruction bundles
+; CHECK-NEXT: Basic Block Hash Compute
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: X86 Assembly Printer
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
index 63779727ec72c..dd97aecd57cf3 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
@@ -81,26 +81,32 @@ declare i32 @__gxx_personality_v0(...)
; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0
; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0
; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 1 # BB id
; CHECK-NEXT: .uleb128 .LBB0_1-.LBB_END0_0
; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1
; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 3 # BB id
; CHECK-NEXT: .uleb128 .LBB0_2-.LBB_END0_1
; CHECK-NEXT: .uleb128 .LBB_END0_2-.LBB0_2
; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 5 # BB id
; CHECK-NEXT: .uleb128 .LBB0_3-.LBB_END0_2
; CHECK-NEXT: .uleb128 .LBB_END0_3-.LBB0_3
; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 4 # BB id
; CHECK-NEXT: .uleb128 .LBB0_4-.LBB_END0_3
; CHECK-NEXT: .uleb128 .LBB_END0_4-.LBB0_4
; CHECK-NEXT: .byte 16
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 2 # BB id
; CHECK-NEXT: .uleb128 .LBB0_5-.LBB_END0_4
; CHECK-NEXT: .uleb128 .LBB_END0_5-.LBB0_5
; CHECK-NEXT: .byte 4
+; CHECK-NEXT: .ascii "{{.*}}"
;; PGO Analysis Map
; PGO-NONE-NOT: .byte 100 # function entry count
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
index b897cf4853cac..4f13ad46838ea 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
@@ -54,18 +54,21 @@ declare i32 @__gxx_personality_v0(...)
; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0
; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0
; CHECK-NEXT: .byte 0
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 2 # BB id
; CHECK-NEXT: .uleb128 .LBB0_1-.LBB_END0_0
; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1
; CHECK-NEXT: .byte 5
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .quad _Z3bazb.cold # base address
; CHECK-NEXT: .byte 2 # number of basic blocks
; CHECK-NEXT: .byte 1 # BB id
; CHECK-NEXT: .uleb128 _Z3bazb.cold-_Z3bazb.cold
; CHECK-NEXT: .uleb128 .LBB_END0_2-_Z3bazb.cold
; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 3 # BB id
; CHECK-NEXT: .uleb128 .LBB0_3-.LBB_END0_2
; CHECK-NEXT: .uleb128 .LBB_END0_3-.LBB0_3
; CHECK-NEXT: .byte 1
-
+; CHECK-NEXT: .ascii "{{.*}}"
\ No newline at end of file
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
index 2565db23c0249..304233d0865f4 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
@@ -64,16 +64,19 @@ declare i32 @qux()
; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0
; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0
; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 1 # BB id
; CHECK-NEXT: .uleb128 .LBB0_1-.LBB_END0_0
; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1
; CHECK-NEXT: .byte 3
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .quad foo.cold # base address
; CHECK-NEXT: .byte 1 # number of basic blocks
; CHECK-NEXT: .byte 2 # BB id
; CHECK-NEXT: .uleb128 foo.cold-foo.cold
; CHECK-NEXT: .uleb128 .LBB_END0_2-foo.cold
; CHECK-NEXT: .byte 3
+; CHECK-NEXT: .ascii "{{.*}}"
;; PGO Analysis Map
; PGO: .ascii "\3306" # function entry count
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map.ll b/llvm/test/CodeGen/X86/basic-block-address-map.ll
index 4f12258eeeea0..232488a0eeefc 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map.ll
@@ -57,24 +57,30 @@ declare i32 @__gxx_personality_v0(...)
; CHECK-NEXT: .byte 0 # BB id
; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0
; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0
-; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .byte
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 1 # BB id
; CHECK-NEXT: .uleb128 .LBB0_1-.LBB_END0_0
; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1
; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 3 # BB id
; CHECK-NEXT: .uleb128 .LBB0_2-.LBB_END0_1
; CHECK-NEXT: .uleb128 .LBB_END0_2-.LBB0_2
; CHECK-NEXT: .byte 8
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 4 # BB id
; CHECK-NEXT: .uleb128 .LBB0_3-.LBB_END0_2
; CHECK-NEXT: .uleb128 .LBB_END0_3-.LBB0_3
; CHECK-NEXT: .byte 16
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 5 # BB id
; CHECK-NEXT: .uleb128 .LBB0_4-.LBB_END0_3
; CHECK-NEXT: .uleb128 .LBB_END0_4-.LBB0_4
; CHECK-NEXT: .byte 1
+; CHECK-NEXT: .ascii "{{.*}}"
; CHECK-NEXT: .byte 2 # BB id
; CHECK-NEXT: .uleb128 .LBB0_5-.LBB_END0_4
; CHECK-NEXT: .uleb128 .LBB_END0_5-.LBB0_5
; CHECK-NEXT: .byte 5
+; CHECK-NEXT: .ascii "{{.*}}"
diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
index d72f517cfb603..ff5068634f15d 100644
--- a/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -226,6 +226,7 @@
; CHECK-NEXT: X86 Load Value Injection (LVI) Ret-Hardening
; CHECK-NEXT: Pseudo Probe Inserter
; CHECK-NEXT: Unpack machine instruction bundles
+; CHECK-NEXT: Basic Block Hash Compute
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
; CHECK-NEXT: X86 Assembly Printer
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
index cc7faea67bed2..48ebfec41d9b7 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
+++ b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
@@ -113,28 +113,34 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0x1
+ Hash: 0x1
- ID: 1
AddressOffset: 0x0
Size: 0x6
Metadata: 0x0
+ Hash: 0x2
- ID: 2
AddressOffset: 0x1
Size: 0x4
Metadata: 0x0
+ Hash: 0x3
- ID: 4
AddressOffset: 0x0
Size: 0x6
Metadata: 0x1
+ Hash: 0x4
- ID: 5
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x5
- BaseAddress: 0x6000
BBEntries:
- ID: 6
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x6
- Name: .llvm_bb_addr_map.bar
Type: SHT_LLVM_BB_ADDR_MAP
Link: .text.bar
@@ -146,12 +152,15 @@ Sections:
- AddressOffset: 0x0
Size: 0x1
Metadata: 0x1
+ Hash: 0x7
- AddressOffset: 0x4
Size: 0x2
Metadata: 0x0
+ Hash: 0x8
- AddressOffset: 0x0
Size: 0x6
Metadata: 0x0
+ Hash: 0x9
Symbols:
- Name: foo
@@ -208,28 +217,34 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0x1
+ Hash: 0x1
- ID: 1
AddressOffset: 0x0
Size: 0x6
Metadata: 0x0
+ Hash: 0x2
- ID: 2
AddressOffset: 0x1
Size: 0x4
Metadata: 0x0
+ Hash: 0x3
- ID: 4
AddressOffset: 0x0
Size: 0x6
Metadata: 0x1
+ Hash: 0x4
- ID: 5
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x5
- BaseAddress: 0x6000
BBEntries:
- ID: 6
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x6
- Version: 1
BBRanges:
- BaseAddress: 0x5000
@@ -237,12 +252,15 @@ Sections:
- AddressOffset: 0x0
Size: 0x1
Metadata: 0x1
+ Hash: 0x7
- AddressOffset: 0x4
Size: 0x2
Metadata: 0x0
+ Hash: 0x8
- AddressOffset: 0x0
Size: 0x6
Metadata: 0x0
+ Hash: 0x9
Symbols:
- Name: foo
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-symbolize-relocatable.yaml b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-symbolize-relocatable.yaml
index 706d386e467e7..a32a26adb4a03 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-symbolize-relocatable.yaml
+++ b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-symbolize-relocatable.yaml
@@ -30,6 +30,7 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0xa
+ Hash: 0x1
- Version: 2
BBRanges:
- BBEntries:
@@ -37,6 +38,7 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0xb
+ Hash: 0x2
- Version: 2
Feature: 0x8
BBRanges:
@@ -45,11 +47,13 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0xc
+ Hash: 0x3
- BBEntries:
- ID: 1
AddressOffset: 0x0
Size: 0x1
Metadata: 0xd
+ Hash: 0x4
- Name: .rela.llvm_bb_addr_map
Type: SHT_RELA
Flags: [ SHF_INFO_LINK ]
@@ -59,15 +63,15 @@ Sections:
- Offset: 0x2
Symbol: .text
Type: R_X86_64_64
- - Offset: 0x11
+ - Offset: 0x12
Symbol: .text
Type: R_X86_64_64
Addend: 1
- - Offset: 0x21
+ - Offset: 0x23
Symbol: .text
Type: R_X86_64_64
Addend: 0x2
- - Offset: 0x2e
+ - Offset: 0x31
Symbol: .text
Type: R_X86_64_64
Addend: 0x3
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-pgoanalysismap.yaml b/llvm/test/tools/llvm-objdump/X86/elf-pgoanalysismap.yaml
index 4d1e5408d86d4..c2e202ad02361 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-pgoanalysismap.yaml
+++ b/llvm/test/tools/llvm-objdump/X86/elf-pgoanalysismap.yaml
@@ -33,6 +33,7 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0x1
+ Hash: 0x2
PGOAnalyses:
- FuncEntryCount: 1000
Symbols:
@@ -76,18 +77,22 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0x1
+ Hash: 0x1
- ID: 1
AddressOffset: 0x0
Size: 0x6
Metadata: 0x0
+ Hash: 0x2
- ID: 2
AddressOffset: 0x1
Size: 0x4
Metadata: 0x0
+ Hash: 0x3
- ID: 5
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x4
PGOAnalyses:
- FuncEntryCount: 1000
PGOBBEntries:
@@ -148,18 +153,22 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0x1
+ Hash: 0x1
- ID: 1
AddressOffset: 0x0
Size: 0x6
Metadata: 0x0
+ Hash: 0x2
- ID: 2
AddressOffset: 0x1
Size: 0x4
Metadata: 0x0
+ Hash: 0x3
- ID: 5
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x4
PGOAnalyses:
- FuncEntryCount: 1000
PGOBBEntries:
diff --git a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test
index 5faafd4d83b2f..cbc1c2ca27c7a 100644
--- a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test
+++ b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test
@@ -14,8 +14,8 @@
# RUN: llvm-readelf %t1.x32.o --bb-addr-map | FileCheck %s --check-prefix=GNU
## Check that a malformed section can be handled.
-# RUN: yaml2obj %s -DBITS=32 -DSIZE=24 -o %t2.o
-# RUN: llvm-readobj %t2.o --bb-addr-map 2>&1 | FileCheck --match-full-lines %s -DOFFSET=0x00000018 -DFILE=%t2.o --check-prefix=TRUNCATED
+# RUN: yaml2obj %s -DBITS=32 -DSIZE=27 -o %t2.o
+# RUN: llvm-readobj %t2.o --bb-addr-map 2>&1 | FileCheck --match-full-lines %s -DOFFSET=0x0000001b -DFILE=%t2.o --check-prefix=TRUNCATED
## Check that missing features can be handled.
# RUN: yaml2obj %s -DBITS=32 -DFEATURE=0x2 -o %t3.o
@@ -39,6 +39,7 @@
# CHECK-NEXT: IsEHPad: No
# CHECK-NEXT: CanFallThrough: No
# CHECK-NEXT: HasIndirectBranch: No
+# CHECK-NEXT: Hash: 0x1
# CHECK-NEXT: }
# CHECK-NEXT: {
# CHECK-NEXT: ID: 2
@@ -49,6 +50,7 @@
# CHECK-NEXT: IsEHPad: Yes
# CHECK-NEXT: CanFallThrough: No
# CHECK-NEXT: HasIndirectBranch: Yes
+# CHECK-NEXT: Hash: 0x2
# CHECK-NEXT: }
# CHECK-NEXT: ]
# CHECK-NEXT: }
@@ -92,6 +94,7 @@
# CHECK-NEXT: IsEHPad: No
# CHECK-NEXT: CanFallThrough: Yes
# CHECK-NEXT: HasIndirectBranch: No
+# CHECK-NEXT: Hash: 0x3
# CHECK-NEXT: }
# CHECK-NEXT: ]
# CHECK-NEXT: }
@@ -133,6 +136,7 @@
# TRUNCATED-NEXT: IsEHPad: No
# TRUNCATED-NEXT: CanFallThrough: Yes
# TRUNCATED-NEXT: HasIndirectBranch: Yes
+# TRUNCATED-NEXT: Hash: 0x4
# TRUNCATED-NEXT: }
# TRUNCATED-NEXT: {
# TRUNCATED-NEXT: ID: 7
@@ -143,6 +147,7 @@
# TRUNCATED-NEXT: IsEHPad: Yes
# TRUNCATED-NEXT: CanFallThrough: Yes
# TRUNCATED-NEXT: HasIndirectBranch: No
+# TRUNCATED-NEXT: Hash: 0x5
# TRUNCATED-NEXT: }
# TRUNCATED-NEXT: ]
# TRUNCATED-NEXT: }
@@ -153,7 +158,7 @@
# TRUNCATED-NEXT: }
# TRUNCATED-NEXT: ]
-# INVALIDFT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 5: unable to decode LEB128 at offset 0x00000010: malformed uleb128, extends past end
+# INVALIDFT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 5: unable to decode LEB128 at offset 0x00000012: malformed uleb128, extends past end
--- !ELF
FileHeader:
@@ -181,10 +186,12 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x1
- ID: 2
AddressOffset: 0x3
Size: 0x4
Metadata: 0x15
+ Hash: 0x2
- Version: 2
Feature: 0x3
BBRanges:
@@ -194,6 +201,7 @@ Sections:
AddressOffset: 0x6
Size: 0x7
Metadata: 0x8
+ Hash: 0x3
PGOAnalyses:
- FuncEntryCount: 100
PGOBBEntries:
@@ -222,10 +230,12 @@ Sections:
AddressOffset: 0x9
Size: 0xa
Metadata: 0x1b
+ Hash: 0x4
- ID: 7
AddressOffset: 0xc
Size: 0xd
Metadata: 0xe
+ Hash: 0x5
PGOAnalyses:
- FuncEntryCount: 89
Symbols:
diff --git a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test
index 325a956e78591..e0847fd2de68d 100644
--- a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test
+++ b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map-relocatable.test
@@ -22,6 +22,7 @@
# CHECK-NEXT: IsEHPad: No
# CHECK-NEXT: CanFallThrough: No
# CHECK-NEXT: HasIndirectBranch: No
+# CHECK-NEXT: Hash: 0x1
# CHECK-NEXT: }
# CHECK-NEXT: ]
# CHECK-NEXT: }
@@ -43,6 +44,7 @@
# CHECK-NEXT: IsEHPad: No
# CHECK-NEXT: CanFallThrough: Yes
# CHECK-NEXT: HasIndirectBranch: No
+# CHECK-NEXT: Hash: 0x2
# CHECK-NEXT: }
# CHECK-NEXT: ]
# CHECK-NEXT: }
@@ -71,6 +73,7 @@ Sections:
AddressOffset: 0x0
Size: 0xF
Metadata: 0x1
+ Hash: 0x1
- Version: 2
BBRanges:
- BBEntries:
@@ -78,6 +81,7 @@ Sections:
AddressOffset: 0x0
Size: 0x11
Metadata: 0x8
+ Hash: 0x2
- Name: [[RELOCATION_SECTION_NAME]]
Type: [[RELOCATION_SECTION_TYPE]]
Flags: [ SHF_INFO_LINK ]
@@ -87,7 +91,7 @@ Sections:
- Offset: 0x2
Symbol: .text
Type: R_X86_64_64
- - Offset: 0x11
+ - Offset: 0x12
Symbol: .text
Type: R_X86_64_64
Addend: 16
@@ -147,6 +151,7 @@ Sections:
AddressOffset: 0x0
Size: 0xF
Metadata: 0x1
+ Hash: 0x1
- Name: .rela.llvm_bb_addr_map
Type: SHT_RELA
Flags: [ SHF_INFO_LINK ]
@@ -202,6 +207,7 @@ Sections:
AddressOffset: 0x0
Size: 0xF
Metadata: 0x1
+ Hash: 0x1
# RUN: yaml2obj %s --docnum=5 -o %t5.o
# RUN: llvm-readobj %t5.o --bb-addr-map 2>&1 | FileCheck %s --check-prefix=ET-DYN-NO-WARNING -DFILE=%t5.o
@@ -223,6 +229,7 @@ Sections:
# ET-DYN-NO-WARNING: IsEHPad: No
# ET-DYN-NO-WARNING: CanFallThrough: No
# ET-DYN-NO-WARNING: HasIndirectBranch: No
+# ET-DYN-NO-WARNING: Hash: 0x1
# ET-DYN-NO-WARNING: }
# ET-DYN-NO-WARNING: ]
# ET-DYN-NO-WARNING: }
@@ -254,6 +261,7 @@ Sections:
# CREL-NEXT: IsEHPad: No
# CREL-NEXT: CanFallThrough: No
# CREL-NEXT: HasIndirectBranch: No
+# CREL-NEXT: Hash: 0x1
# CREL-NEXT: }
# CREL-NEXT: ]
# CREL-NEXT: }
@@ -275,6 +283,7 @@ Sections:
# CREL-NEXT: IsEHPad: No
# CREL-NEXT: CanFallThrough: Yes
# CREL-NEXT: HasIndirectBranch: No
+# CREL-NEXT: Hash: 0x2
# CREL-NEXT: }
# CREL-NEXT: ]
# CREL-NEXT: }
diff --git a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test
index c5d071c11d1de..3b6b49ebf0396 100644
--- a/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test
+++ b/llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test
@@ -39,7 +39,8 @@
# CHECK-NEXT: HasTailCall: Yes
# CHECK-NEXT: IsEHPad: No
# CHECK-NEXT: CanFallThrough: No
-# CHECK-NEXT: HasIndirectBranch: No
+# CHECK-NEXT: HasIndirectBranch:
+# CHECK-NEXT: Hash: 0x1
# CHECK-NEXT: }
# CHECK-NEXT: ]
# CHECK-NEXT: }
@@ -55,6 +56,7 @@
# CHECK-NEXT: IsEHPad: Yes
# CHECK-NEXT: CanFallThrough: No
# CHECK-NEXT: HasIndirectBranch: Yes
+# CHECK-NEXT: Hash: 0x2
# CHECK-NEXT: }
# CHECK-NEXT: ]
# CHECK-NEXT: }
@@ -75,7 +77,8 @@
# CHECK-NEXT: HasTailCall: No
# CHECK-NEXT: IsEHPad: No
# CHECK-NEXT: CanFallThrough: Yes
-# CHECK-NEXT: HasIndirectBranch: No
+# CHECK-NEXT: HasIndirectBranch: No
+# CHECK-NEXT: Hash: 0x3
# CHECK-NEXT: }
# CHECK-NEXT: ]
# CHECK-NEXT: }
@@ -106,6 +109,7 @@
# TRUNCATED-NEXT: IsEHPad: No
# TRUNCATED-NEXT: CanFallThrough: Yes
# TRUNCATED-NEXT: HasIndirectBranch: Yes
+# TRUNCATED-NEXT: Hash: 0x4
# TRUNCATED-NEXT: }
# TRUNCATED-NEXT: {
# TRUNCATED-NEXT: ID: 7
@@ -116,6 +120,7 @@
# TRUNCATED-NEXT: IsEHPad: Yes
# TRUNCATED-NEXT: CanFallThrough: Yes
# TRUNCATED-NEXT: HasIndirectBranch: No
+# TRUNCATED-NEXT: Hash: 0x5
# TRUNCATED-NEXT: }
# TRUNCATED-NEXT: ]
# TRUNCATED-NEXT: }
@@ -152,12 +157,14 @@ Sections:
AddressOffset: 0x0
Size: 0x1
Metadata: [[METADATA=0x2]]
+ Hash: 0x1
- BaseAddress: 0x44444
BBEntries:
- ID: 2
AddressOffset: 0x3
Size: 0x4
Metadata: 0x15
+ Hash: 0x2
- Version: 2
BBRanges:
- BaseAddress: 0x22222
@@ -166,6 +173,7 @@ Sections:
AddressOffset: 0x6
Size: 0x7
Metadata: 0x8
+ Hash: 0x3
- Name: dummy_section
Type: SHT_PROGBITS
Size: 16
@@ -181,10 +189,12 @@ Sections:
AddressOffset: 0x9
Size: 0xa
Metadata: 0x1b
+ Hash: 0x4
- ID: 7
AddressOffset: 0xc
Size: 0xd
Metadata: 0xe
+ Hash: 0x5
Symbols:
- Name: foo
Section: .text
diff --git a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
index 299bf463cf4bc..7e2c8d49726c8 100644
--- a/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/bb-addr-map-pgo-analysis-map.yaml
@@ -23,14 +23,17 @@
# VALID-NEXT: AddressOffset: 0x1
# VALID-NEXT: Size: 0x2
# VALID-NEXT: Metadata: 0x3
+# VALID-NEXT: Hash: 0x4
# VALID-NEXT: - ID: 2
# VALID-NEXT: AddressOffset: 0x4
# VALID-NEXT: Size: 0x5
# VALID-NEXT: Metadata: 0x6
+# VALID-NEXT: Hash: 0x7
# VALID-NEXT: - ID: 4
# VALID-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
# VALID-NEXT: Size: 0xFFFFFFFFFFFFFFF8
# VALID-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
+# VALID-NEXT: Hash: 0xFFFFFFFFFFFFFFFA
# VALID-NEXT: - Version: 2
# VALID-NEXT: Feature: 0xA
# VALID-NEXT: BBRanges:
@@ -40,6 +43,7 @@
# VALID-NEXT: AddressOffset: 0xA
# VALID-NEXT: Size: 0xB
# VALID-NEXT: Metadata: 0xC
+# VALID-NEXT: Hash: 0xD
# VALID-NEXT: PGOAnalyses:
# VALID-NEXT: - FuncEntryCount: 100
# VALID-NEXT: PGOBBEntries:
@@ -77,14 +81,17 @@ Sections:
AddressOffset: 0x1
Size: 0x2
Metadata: 0x3
+ Hash: 0x4
- ID: 2
AddressOffset: 0x4
Size: 0x5
Metadata: 0x6
+ Hash: 0x7
- ID: 4
AddressOffset: 0xFFFFFFFFFFFFFFF7
Size: 0xFFFFFFFFFFFFFFF8
Metadata: 0xFFFFFFFFFFFFFFF9
+ Hash: 0xFFFFFFFFFFFFFFFA
- Version: 2
Feature: 0xA
BBRanges:
@@ -94,6 +101,7 @@ Sections:
AddressOffset: 0xA
Size: 0xB
Metadata: 0xC
+ Hash: 0xD
PGOAnalyses:
- FuncEntryCount: 100
PGOBBEntries:
@@ -135,6 +143,7 @@ Sections:
# MULTI-NEXT: AddressOffset: 0x1
# MULTI-NEXT: Size: 0x2
# MULTI-NEXT: Metadata: 0x3
+# MULTI-NEXT: Hash: 0x4
# MULTI-NEXT: PGOAnalyses:
# MULTI-NEXT: - FuncEntryCount: 0
# MULTI-NEXT: PGOBBEntries:
@@ -169,6 +178,7 @@ Sections:
- AddressOffset: 0x1
Size: 0x2
Metadata: 0x3
+ Hash: 0x4
PGOAnalyses:
- FuncEntryCount: 0
PGOBBEntries:
diff --git a/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml b/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
index 8dbf97ef2bc12..5c995dc88dac0 100644
--- a/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml
@@ -22,14 +22,17 @@
# VALID-NEXT: AddressOffset: 0x1
# VALID-NEXT: Size: 0x2
# VALID-NEXT: Metadata: 0x3
+# VALID-NEXT: Hash: 0x1
# VALID-NEXT: - ID: 2
# VALID-NEXT: AddressOffset: 0x4
# VALID-NEXT: Size: 0x5
# VALID-NEXT: Metadata: 0x6
+# VALID-NEXT: Hash: 0x2
# VALID-NEXT: - ID: 4
# VALID-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
# VALID-NEXT: Size: 0xFFFFFFFFFFFFFFF8
# VALID-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
+# VALID-NEXT: Hash: 0xFFFFFFFFFFFFFFFA
# VALID-NEXT: - Version: 2
# VALID-NEXT: Feature: 0x8
# VALID-NEXT: BBRanges:
@@ -39,6 +42,7 @@
# VALID-NEXT: AddressOffset: 0xA
# VALID-NEXT: Size: 0xB
# VALID-NEXT: Metadata: 0xC
+# VALID-NEXT: Hash: 0xD
--- !ELF
FileHeader:
@@ -59,14 +63,17 @@ Sections:
AddressOffset: 0x1
Size: 0x2
Metadata: 0x3
+ Hash: 0x1
- ID: 2
AddressOffset: 0x4
Size: 0x5
Metadata: 0x6
+ Hash: 0x2
- ID: 4
AddressOffset: 0xFFFFFFFFFFFFFFF7
Size: 0xFFFFFFFFFFFFFFF8
Metadata: 0xFFFFFFFFFFFFFFF9
+ Hash: 0xFFFFFFFFFFFFFFFA
- Version: 2
Feature: 0x8
NumBBRanges: [[NUMBBRANGES=<none>]]
@@ -78,6 +85,7 @@ Sections:
AddressOffset: 0xA
Size: 0xB
Metadata: 0xC
+ Hash: 0xD
## Check obj2yaml can dump empty .llvm_bb_addr_map sections.
@@ -126,6 +134,7 @@ Sections:
# MULTI-NEXT: AddressOffset: 0x1
# MULTI-NEXT: Size: 0x2
# MULTI-NEXT: Metadata: 0x3
+# MULTI-NEXT: Hash: 0x4
# MULTI-NEXT: - Name: '.llvm_bb_addr_map (1)'
# MULTI-NEXT: Type: SHT_LLVM_BB_ADDR_MAP
# MULTI-NEXT: Entries:
@@ -153,6 +162,7 @@ Sections:
- AddressOffset: 0x1
Size: 0x2
Metadata: 0x3
+ Hash: 0x4
- Name: '.llvm_bb_addr_map (1)'
Type: SHT_LLVM_BB_ADDR_MAP
Entries:
diff --git a/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml b/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml
index a4cb572e6d993..29e190ac03342 100644
--- a/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/bb-addr-map-pgo-analysis-map.yaml
@@ -6,14 +6,15 @@
# Case 4: Specify Entries.
# CHECK: Name: .llvm_bb_addr_map (1)
# CHECK: SectionData (
-# CHECK-NEXT: 0000: 02072000 00000000 0000010B 010203E8
-# CHECK-NEXT: 0010: 07E80702 0CEEDDBB F70E0D91 A2C48801
+# CHECK-NEXT: 0000: 02072000 00000000 0000010B 01020304
+# CHECK-NEXT: 0010: E807E807 020CEEDD BBF70E0D 91A2C488
+# CHECK-NEXT: 0020: 01
# CHECK-NEXT: )
# Case 7: Not including a field which is enabled in feature doesn't emit value
# CHECK: Name: .llvm_bb_addr_map (1)
# CHECK: SectionData (
-# CHECK-NEXT: 0000: 02012000 00000000 0000020D 010203 |
+# CHECK-NEXT: 0000: 02012000 00000000 0000020D 01020304 |
# CHECK-NEXT: )
--- !ELF
@@ -39,6 +40,7 @@ Sections:
AddressOffset: 0x00000001
Size: 0x00000002
Metadata: 0x00000003
+ Hash: 0x00000004
PGOAnalyses:
- FuncEntryCount: 1000
PGOBBEntries:
@@ -63,6 +65,7 @@ Sections:
AddressOffset: 0x00000001
Size: 0x00000002
Metadata: 0x00000003
+ Hash: 0x00000004
## Check that yaml2obj generates a warning when we use unsupported feature.
# RUN: yaml2obj --docnum=2 %s 2>&1 | FileCheck %s --check-prefix=INVALID-FEATURE
diff --git a/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml b/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml
index 709938babffbf..bc18bc266de50 100644
--- a/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/bb-addr-map.yaml
@@ -36,7 +36,7 @@
# Case 4: Specify Entries.
# CHECK: Name: .llvm_bb_addr_map (1)
# CHECK: SectionData (
-# CHECK-NEXT: 0000: 02002000 00000000 0000010B 010203
+# CHECK-NEXT: 0000: 02002000 00000000 0000010B 01020304
# CHECK-NEXT: )
# Case 5: Specify Entries and omit the Address field.
@@ -44,13 +44,13 @@
# CHECK: Address:
# CHECK-SAME: {{^ 0x0$}}
# CHECK: SectionData (
-# CHECK-NEXT: 0000: 02000000 00000000 0000010C 010203
+# CHECK-NEXT: 0000: 02000000 00000000 0000010C 01020304
# CHECK-NEXT: )
# Case 6: Override the NumBlocks field.
# CHECK: Name: .llvm_bb_addr_map (1)
# CHECK: SectionData (
-# CHECK-NEXT: 0000: 02002000 00000000 0000020D 010203
+# CHECK-NEXT: 0000: 02002000 00000000 0000020D 01020304
# CHECK-NEXT: )
# Case 7: Specify empty BBRanges.
@@ -108,6 +108,7 @@ Sections:
AddressOffset: 0x00000001
Size: 0x00000002
Metadata: 0x00000003
+ Hash: 0x00000004
## 5) When specifying the description with Entries, the 'Address' field will be
## zero when omitted.
@@ -121,6 +122,7 @@ Sections:
AddressOffset: 0x00000001
Size: 0x00000002
Metadata: 0x00000003
+ Hash: 0x00000004
## 6) We can override the NumBlocks field with a value different from the
## actual number of BB Entries.
@@ -136,6 +138,7 @@ Sections:
AddressOffset: 0x00000001
Size: 0x00000002
Metadata: 0x00000003
+ Hash: 0x00000004
## 7) We can produce a SHT_LLVM_BB_ADDR_MAP section from a description
## with one entry with empty BBRanges.
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index d01cab211afd7..cfd853cd0efae 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -7899,6 +7899,7 @@ void LLVMELFDumper<ELFT>::printBBAddrMaps(bool PrettyPGOAnalysis) {
W.printBoolean("IsEHPad", BBE.isEHPad());
W.printBoolean("CanFallThrough", BBE.canFallThrough());
W.printBoolean("HasIndirectBranch", BBE.hasIndirectBranch());
+ W.printHex("Hash", BBE.Hash);
}
}
}
diff --git a/llvm/tools/obj2yaml/elf2yaml.cpp b/llvm/tools/obj2yaml/elf2yaml.cpp
index c56ed15501b40..e53721d63f120 100644
--- a/llvm/tools/obj2yaml/elf2yaml.cpp
+++ b/llvm/tools/obj2yaml/elf2yaml.cpp
@@ -936,7 +936,8 @@ ELFDumper<ELFT>::dumpBBAddrMapSection(const Elf_Shdr *Shdr) {
uint64_t Offset = Data.getULEB128(Cur);
uint64_t Size = Data.getULEB128(Cur);
uint64_t Metadata = Data.getULEB128(Cur);
- BBEntries.push_back({ID, Offset, Size, Metadata});
+ uint64_t Hash = Data.getULEB128(Cur);
+ BBEntries.push_back({ID, Offset, Size, Metadata, Hash});
}
TotalNumBlocks += BBEntries.size();
BBRanges.push_back({BaseAddress, /*NumBlocks=*/{}, BBEntries});
diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp
index 493e673d6a07d..fa5611ed4174b 100644
--- a/llvm/unittests/Object/ELFObjectFileTest.cpp
+++ b/llvm/unittests/Object/ELFObjectFileTest.cpp
@@ -538,6 +538,7 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) {
- AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x1
)";
{
@@ -568,6 +569,7 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) {
- AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x1
)";
// Check that we can detect the malformed encoding when the section is
@@ -592,6 +594,7 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) {
AddressOffset: 0x100000000
Size: 0xFFFFFFFF
Metadata: 0xFFFFFFFF
+ Hash: 0xFFFFFFFF
)";
OverInt32LimitYamlStrings[1] += R"(
@@ -599,6 +602,7 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) {
AddressOffset: 0xFFFFFFFF
Size: 0x100000000
Metadata: 0xFFFFFFFF
+ Hash: 0xFFFFFFFF
)";
OverInt32LimitYamlStrings[2] += R"(
@@ -606,16 +610,17 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) {
AddressOffset: 0xFFFFFFFF
Size: 0xFFFFFFFF
Metadata: 0x100000000
+ Hash: 0xFFFFFFFF
)";
{
SCOPED_TRACE("overlimit fields");
DoCheck(OverInt32LimitYamlStrings[0],
- "ULEB128 value at offset 0x10 exceeds UINT32_MAX (0x100000000)");
+ "ULEB128 value at offset 0x11 exceeds UINT32_MAX (0x100000000)");
DoCheck(OverInt32LimitYamlStrings[1],
- "ULEB128 value at offset 0x15 exceeds UINT32_MAX (0x100000000)");
+ "ULEB128 value at offset 0x16 exceeds UINT32_MAX (0x100000000)");
DoCheck(OverInt32LimitYamlStrings[2],
- "ULEB128 value at offset 0x1a exceeds UINT32_MAX (0x100000000)");
+ "ULEB128 value at offset 0x1b exceeds UINT32_MAX (0x100000000)");
}
// Check the proper error handling when the section has fields exceeding
@@ -625,26 +630,26 @@ TEST(ELFObjectFileTest, InvalidDecodeBBAddrMap) {
3, OverInt32LimitYamlStrings[1]);
// Truncate before the end of the 5-byte field.
OverInt32LimitAndTruncated[0] += R"(
- ShSize: 0x19
+ ShSize: 0x1a
)";
// Truncate at the end of the 5-byte field.
OverInt32LimitAndTruncated[1] += R"(
- ShSize: 0x1a
+ ShSize: 0x1b
)";
// Truncate after the end of the 5-byte field.
OverInt32LimitAndTruncated[2] += R"(
- ShSize: 0x1b
+ ShSize: 0x1c
)";
{
SCOPED_TRACE("overlimit fields, truncated section");
DoCheck(OverInt32LimitAndTruncated[0],
- "unable to decode LEB128 at offset 0x00000015: malformed uleb128, "
+ "unable to decode LEB128 at offset 0x00000016: malformed uleb128, "
"extends past end");
DoCheck(OverInt32LimitAndTruncated[1],
- "ULEB128 value at offset 0x15 exceeds UINT32_MAX (0x100000000)");
+ "ULEB128 value at offset 0x16 exceeds UINT32_MAX (0x100000000)");
DoCheck(OverInt32LimitAndTruncated[2],
- "ULEB128 value at offset 0x15 exceeds UINT32_MAX (0x100000000)");
+ "ULEB128 value at offset 0x16 exceeds UINT32_MAX (0x100000000)");
}
// Check for proper error handling when the 'NumBlocks' field is overridden
@@ -692,6 +697,7 @@ TEST(ELFObjectFileTest, ReadBBAddrMap) {
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x4
- Name: .llvm_bb_addr_map_2
Type: SHT_LLVM_BB_ADDR_MAP
Link: 1
@@ -705,12 +711,14 @@ TEST(ELFObjectFileTest, ReadBBAddrMap) {
AddressOffset: 0x0
Size: 0x2
Metadata: 0x4
+ Hash: 0x5
- BaseAddress: 0xFFFFF
BBEntries:
- ID: 15
AddressOffset: 0xF0
Size: 0xF1
Metadata: 0x1F
+ Hash: 0x6
- Name: .llvm_bb_addr_map_3
Type: SHT_LLVM_BB_ADDR_MAP
Link: 2
@@ -723,6 +731,7 @@ TEST(ELFObjectFileTest, ReadBBAddrMap) {
AddressOffset: 0x0
Size: 0x3
Metadata: 0x6
+ Hash: 0x7
- Name: .llvm_bb_addr_map_4
Type: SHT_LLVM_BB_ADDR_MAP
# Link: 0 (by default, can be overriden)
@@ -735,17 +744,18 @@ TEST(ELFObjectFileTest, ReadBBAddrMap) {
AddressOffset: 0x0
Size: 0x4
Metadata: 0x18
+ Hash: 0x8
)");
BBAddrMap E1 = {
- {{0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}}}}}};
+ {{0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}, 0x4}}}}};
BBAddrMap E2 = {
- {{0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}}}},
- {0xFFFFF, {{15, 0xF0, 0xF1, {true, true, true, true, true}}}}}};
+ {{0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}, 0x5}}},
+ {0xFFFFF, {{15, 0xF0, 0xF1, {true, true, true, true, true}, 0x6}}}}};
BBAddrMap E3 = {
- {{0x33333, {{0, 0x0, 0x3, {false, true, true, false, false}}}}}};
+ {{0x33333, {{0, 0x0, 0x3, {false, true, true, false, false}, 0x7}}}}};
BBAddrMap E4 = {
- {{0x44444, {{0, 0x0, 0x4, {false, false, false, true, true}}}}}};
+ {{0x44444, {{0, 0x0, 0x4, {false, false, false, true, true}, 0x8}}}}};
std::vector<BBAddrMap> Section0BBAddrMaps = {E4};
std::vector<BBAddrMap> Section1BBAddrMaps = {E3};
@@ -887,6 +897,7 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
- AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x1
)";
{
@@ -905,6 +916,7 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x1
)";
// Check that we fail when function entry count is enabled but not provided.
@@ -931,11 +943,12 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x1
)";
{
SCOPED_TRACE("missing bb frequency");
- DoCheck(MissingBBFreq, "unable to decode LEB128 at offset 0x0000000f: "
+ DoCheck(MissingBBFreq, "unable to decode LEB128 at offset 0x00000010: "
"malformed uleb128, extends past end");
}
@@ -950,14 +963,17 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x1
Metadata: 0x6
+ Hash: 0x1
- ID: 2
AddressOffset: 0x1
Size: 0x1
Metadata: 0x2
+ Hash: 0x2
- ID: 3
AddressOffset: 0x2
Size: 0x1
Metadata: 0x2
+ Hash: 0x3
PGOAnalyses:
- PGOBBEntries:
- Successors:
@@ -972,7 +988,7 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
{
SCOPED_TRACE("missing branch probability");
- DoCheck(MissingBrProb, "unable to decode LEB128 at offset 0x00000017: "
+ DoCheck(MissingBrProb, "unable to decode LEB128 at offset 0x0000001a: "
"malformed uleb128, extends past end");
}
}
@@ -999,6 +1015,7 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x1
Metadata: 0x2
+ Hash: 0x1
PGOAnalyses:
- FuncEntryCount: 892
- Name: .llvm_bb_addr_map_2
@@ -1014,6 +1031,7 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x2
Metadata: 0x4
+ Hash: 0x2
PGOAnalyses:
- PGOBBEntries:
- BBFreq: 343
@@ -1030,14 +1048,17 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x3
Metadata: 0x6
+ Hash: 0x3
- ID: 1
AddressOffset: 0x0
Size: 0x3
Metadata: 0x4
+ Hash: 0x4
- ID: 2
AddressOffset: 0x0
Size: 0x3
Metadata: 0x0
+ Hash: 0x5
PGOAnalyses:
- PGOBBEntries:
- Successors:
@@ -1062,18 +1083,22 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x4
Metadata: 0x18
+ Hash: 0x6
- ID: 1
AddressOffset: 0x0
Size: 0x4
Metadata: 0x0
+ Hash: 0x7
- ID: 2
AddressOffset: 0x0
Size: 0x4
Metadata: 0x0
+ Hash: 0x8
- ID: 3
AddressOffset: 0x0
Size: 0x4
Metadata: 0x0
+ Hash: 0x9
PGOAnalyses:
- FuncEntryCount: 1000
PGOBBEntries:
@@ -1110,6 +1135,7 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x2
Metadata: 0x4
+ Hash: 0xa
PGOAnalyses: [{}]
- Name: .llvm_bb_addr_map_6
Type: SHT_LLVM_BB_ADDR_MAP
@@ -1124,16 +1150,19 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
AddressOffset: 0x0
Size: 0x6
Metadata: 0x6
+ Hash: 0xb
- ID: 1
AddressOffset: 0x0
Size: 0x6
Metadata: 0x4
+ Hash: 0xc
- BaseAddress: 0x666661
BBEntries:
- ID: 2
AddressOffset: 0x0
Size: 0x6
Metadata: 0x0
+ Hash: 0xd
PGOAnalyses:
- PGOBBEntries:
- Successors:
@@ -1148,16 +1177,16 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
)");
BBAddrMap E1 = {
- {{0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}}}}}};
+ {{0x11111, {{1, 0x0, 0x1, {false, true, false, false, false}, 0x1}}}}};
PGOAnalysisMap P1 = {892, {}, {true, false, false, false, false}};
BBAddrMap E2 = {
- {{0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}}}}}};
+ {{0x22222, {{2, 0x0, 0x2, {false, false, true, false, false}, 0x2}}}}};
PGOAnalysisMap P2 = {
{}, {{BlockFrequency(343), {}}}, {false, true, false, false, false}};
BBAddrMap E3 = {{{0x33333,
- {{0, 0x0, 0x3, {false, true, true, false, false}},
- {1, 0x3, 0x3, {false, false, true, false, false}},
- {2, 0x6, 0x3, {false, false, false, false, false}}}}}};
+ {{0, 0x0, 0x3, {false, true, true, false, false}, 0x3},
+ {1, 0x3, 0x3, {false, false, true, false, false}, 0x4},
+ {2, 0x6, 0x3, {false, false, false, false, false}, 0x5}}}}};
PGOAnalysisMap P3 = {{},
{{{},
{{1, BranchProbability::getRaw(0x1111'1111)},
@@ -1166,10 +1195,10 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
{{}, {}}},
{false, false, true, false, false}};
BBAddrMap E4 = {{{0x44444,
- {{0, 0x0, 0x4, {false, false, false, true, true}},
- {1, 0x4, 0x4, {false, false, false, false, false}},
- {2, 0x8, 0x4, {false, false, false, false, false}},
- {3, 0xc, 0x4, {false, false, false, false, false}}}}}};
+ {{0, 0x0, 0x4, {false, false, false, true, true}, 0x6},
+ {1, 0x4, 0x4, {false, false, false, false, false}, 0x7},
+ {2, 0x8, 0x4, {false, false, false, false, false}, 0x8},
+ {3, 0xc, 0x4, {false, false, false, false, false}, 0x9}}}}};
PGOAnalysisMap P4 = {
1000,
{{BlockFrequency(1000),
@@ -1183,13 +1212,13 @@ TEST(ELFObjectFileTest, ReadPGOAnalysisMap) {
{BlockFrequency(1000), {}}},
{true, true, true, false, false}};
BBAddrMap E5 = {
- {{0x55555, {{2, 0x0, 0x2, {false, false, true, false, false}}}}}};
+ {{0x55555, {{2, 0x0, 0x2, {false, false, true, false, false}, 0xa}}}}};
PGOAnalysisMap P5 = {{}, {}, {false, false, false, false, false}};
BBAddrMap E6 = {
{{0x66666,
- {{0, 0x0, 0x6, {false, true, true, false, false}},
- {1, 0x6, 0x6, {false, false, true, false, false}}}},
- {0x666661, {{2, 0x0, 0x6, {false, false, false, false, false}}}}}};
+ {{0, 0x0, 0x6, {false, true, true, false, false}, 0xb},
+ {1, 0x6, 0x6, {false, false, true, false, false}, 0xc}}},
+ {0x666661, {{2, 0x0, 0x6, {false, false, false, false, false}, 0xd}}}}};
PGOAnalysisMap P6 = {{},
{{{},
{{1, BranchProbability::getRaw(0x2222'2222)},
>From f0cc969b72b33291bdc2847d81789b20211de1cf Mon Sep 17 00:00:00 2001
From: wudexin <wudexin at kuaishou.com>
Date: Mon, 28 Apr 2025 14:44:32 +0800
Subject: [PATCH 2/2] Adding Matching and Inference Functionality to Propeller
---
clang/include/clang/Basic/CodeGenOptions.h | 3 +
clang/include/clang/Driver/Options.td | 5 +
clang/lib/CodeGen/BackendUtil.cpp | 28 ++-
clang/lib/Driver/ToolChains/Clang.cpp | 12 ++
lld/ELF/Config.h | 1 +
lld/ELF/Driver.cpp | 77 +++++++
lld/ELF/LTO.cpp | 1 +
lld/ELF/Options.td | 3 +
.../CodeGen/FuncHotBBHashesProfileReader.h | 64 ++++++
.../HotMachineBasicBlockInfoGenerator.h | 51 +++++
llvm/include/llvm/CodeGen/Passes.h | 4 +
llvm/include/llvm/InitializePasses.h | 2 +
llvm/include/llvm/LTO/Config.h | 3 +
llvm/include/llvm/Support/PGOOptions.h | 2 +
llvm/lib/CodeGen/BasicBlockSections.cpp | 48 ++++-
llvm/lib/CodeGen/CMakeLists.txt | 2 +
llvm/lib/CodeGen/CodeGen.cpp | 2 +
.../CodeGen/FuncHotBBHashesProfileReader.cpp | 123 +++++++++++
.../HotMachineBasicBlockInfoGenerator.cpp | 202 ++++++++++++++++++
llvm/lib/CodeGen/TargetPassConfig.cpp | 13 +-
llvm/lib/LTO/LTOBackend.cpp | 24 ++-
llvm/lib/Support/PGOOptions.cpp | 10 +-
llvm/tools/opt/NewPMDriver.cpp | 23 +-
23 files changed, 668 insertions(+), 35 deletions(-)
create mode 100644 llvm/include/llvm/CodeGen/FuncHotBBHashesProfileReader.h
create mode 100644 llvm/include/llvm/CodeGen/HotMachineBasicBlockInfoGenerator.h
create mode 100644 llvm/lib/CodeGen/FuncHotBBHashesProfileReader.cpp
create mode 100644 llvm/lib/CodeGen/HotMachineBasicBlockInfoGenerator.cpp
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index e39a73bdb13ac..5dbc04f1a1305 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -288,6 +288,9 @@ class CodeGenOptions : public CodeGenOptionsBase {
/// Name of the profile file to use with -fprofile-sample-use.
std::string SampleProfileFile;
+ /// Name of the profile file to use with -fpropeller-profile-use.
+ std::string PropellerProfileFile;
+
/// Name of the profile file to use as output for with -fmemory-profile.
std::string MemoryProfileOutput;
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index affc076a876ad..224abd1b4b2af 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1672,6 +1672,11 @@ def fprofile_sample_accurate : Flag<["-"], "fprofile-sample-accurate">,
as cold. Otherwise, treat callsites without profile samples as if
we have no profile}]>,
MarshallingInfoFlag<CodeGenOpts<"ProfileSampleAccurate">>;
+def fpropeller_profile_use_EQ : Joined<["-"], "fpropeller-profile-use=">,
+ Group<f_Group>, Visibility<[ClangOption, CC1Option, CLOption]>,
+ MetaVarName<"<pathname>">,
+ HelpText<"Use propeller profile for profile-guided optimization">,
+ MarshallingInfoString<CodeGenOpts<"PropellerProfileFile">>;
def fsample_profile_use_profi : Flag<["-"], "fsample-profile-use-profi">,
Visibility<[ClangOption, CC1Option]>,
Group<f_Group>,
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 7557cb8408921..70bb499d4ce60 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -835,7 +835,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
if (CodeGenOpts.hasProfileIRInstr())
// -fprofile-generate.
PGOOpt = PGOOptions(getProfileGenName(CodeGenOpts), "", "",
- CodeGenOpts.MemoryProfileUsePath, nullptr,
+ CodeGenOpts.MemoryProfileUsePath,
+ CodeGenOpts.PropellerProfileFile, nullptr,
PGOOptions::IRInstr, PGOOptions::NoCSAction,
ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling,
/*PseudoProbeForProfiling=*/false,
@@ -846,30 +847,41 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
: PGOOptions::NoCSAction;
PGOOpt = PGOOptions(CodeGenOpts.ProfileInstrumentUsePath, "",
CodeGenOpts.ProfileRemappingFile,
- CodeGenOpts.MemoryProfileUsePath, VFS,
+ CodeGenOpts.MemoryProfileUsePath,
+ CodeGenOpts.PropellerProfileFile, VFS,
PGOOptions::IRUse, CSAction, ClPGOColdFuncAttr,
CodeGenOpts.DebugInfoForProfiling);
} else if (!CodeGenOpts.SampleProfileFile.empty())
// -fprofile-sample-use
PGOOpt = PGOOptions(
CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile,
- CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::SampleUse,
+ CodeGenOpts.MemoryProfileUsePath,
+ CodeGenOpts.PropellerProfileFile, VFS, PGOOptions::SampleUse,
PGOOptions::NoCSAction, ClPGOColdFuncAttr,
CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
else if (!CodeGenOpts.MemoryProfileUsePath.empty())
// -fmemory-profile-use (without any of the above options)
- PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, VFS,
+ PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath,
+ CodeGenOpts.PropellerProfileFile, VFS,
PGOOptions::NoAction, PGOOptions::NoCSAction,
ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling);
+ else if (!CodeGenOpts.PropellerProfileFile.empty())
+ // -fpropeller-profile-use (without any of the above options)
+ PGOOpt = PGOOptions("", "", "", "",
+ CodeGenOpts.PropellerProfileFile, VFS,
+ PGOOptions::NoAction, PGOOptions::NoCSAction,
+ ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling);
else if (CodeGenOpts.PseudoProbeForProfiling)
// -fpseudo-probe-for-profiling
PGOOpt =
- PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
+ PGOOptions("", "", "", /*MemoryProfile=*/"",
+ /*PropellerProfileFile=*/"", nullptr,
PGOOptions::NoAction, PGOOptions::NoCSAction,
ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling, true);
else if (CodeGenOpts.DebugInfoForProfiling)
// -fdebug-info-for-profiling
- PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
+ PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"",
+ /*PropellerProfileFile=*/"", nullptr,
PGOOptions::NoAction, PGOOptions::NoCSAction,
ClPGOColdFuncAttr, true);
@@ -887,7 +899,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
PGOOpt->CSAction = PGOOptions::CSIRInstr;
} else
PGOOpt = PGOOptions("", getProfileGenName(CodeGenOpts), "",
- /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction,
+ /*MemoryProfile=*/"", /*PropellerProfileFile=*/"",
+ nullptr, PGOOptions::NoAction,
PGOOptions::CSIRInstr, ClPGOColdFuncAttr,
CodeGenOpts.DebugInfoForProfiling);
}
@@ -1320,6 +1333,7 @@ runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex,
Conf.OptLevel = CGOpts.OptimizationLevel;
initTargetOptions(CI, Diags, Conf.Options);
Conf.SampleProfile = std::move(SampleProfile);
+ Conf.PropellerProfile = CGOpts.PropellerProfileFile;
Conf.PTO.LoopUnrolling = CGOpts.UnrollLoops;
Conf.PTO.LoopInterchange = CGOpts.InterchangeLoops;
// For historical reasons, loop interleaving is set to mirror setting for loop
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 1de83baacff93..29f5ddc289f21 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6416,6 +6416,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
CmdArgs.push_back("-sample-profile-use-profi");
}
+ if (auto *PropellerUseArg =
+ Args.getLastArg(options::OPT_fpropeller_profile_use_EQ)) {
+ if (Arg *A = Args.getLastArg(options::OPT_fbasic_block_sections_EQ)) {
+ StringRef Val = A->getValue();
+ if (Val.starts_with("list="))
+ D.Diag(diag::err_drv_argument_not_allowed_with)
+ << PropellerUseArg->getAsString(Args)
+ << "-fbasic-block-sections=list";
+ }
+ PropellerUseArg->render(Args, CmdArgs);
+ }
+
// Add runtime flag for PS4/PS5 when PGO, coverage, or sanitizers are enabled.
if (RawTriple.isPS() &&
!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index f0e9592d85dd6..8de8d65b8a0d1 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -211,6 +211,7 @@ struct Config {
llvm::StringRef ltoNewPmPasses;
llvm::StringRef ltoObjPath;
llvm::StringRef ltoSampleProfile;
+ llvm::StringRef ltoPropellerProfile;
llvm::StringRef mapFile;
llvm::StringRef outputFile;
llvm::StringRef optRemarksFilename;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index b3c5518b42877..039457ed65361 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1245,6 +1245,73 @@ static SmallVector<StringRef, 0> getSymbolOrderingFile(Ctx &ctx,
return names.takeVector();
}
+struct HotBBInfo{
+ uint32_t TotalBBSize = 0;
+ uint32_t HotBBSize = 0;
+};
+static SmallVector<StringRef, 0> generateSymbolOrderingFromPropellerProfile(MemoryBufferRef mb) {
+ SmallVector<std::pair<StringRef, HotBBInfo>> names;
+ SmallVector<HotBBInfo *> hotBBInfos;
+ uint32_t line = 0;
+ for (StringRef s : args::getLines(mb)) {
+ line++;
+ if (!s.consume_front("!") || s.empty()) {
+ error("invalid propeller profile at line: " + Twine(line));
+ }
+ if (s.consume_front("!")) {
+ SmallVector<StringRef, 3> HotBB;
+ s.split(HotBB, ' ');
+ if (HotBB.size() != 3)
+ error("invalid propeller profile at line: " + Twine(line));
+ HotBB[0].consume_front("0x");
+ unsigned long long Hash, Freq, BBID;
+ if (getAsUnsignedInteger(HotBB[0], 16, Hash))
+ error("invalid propeller profile at line: " + Twine(line));
+ if (getAsUnsignedInteger(HotBB[1], 10, Freq))
+ error("invalid propeller profile at line: " + Twine(line));
+ if (getAsUnsignedInteger(HotBB[2], 10, BBID))
+ error("invalid propeller profile at line: " + Twine(line));
+ if (Freq > 0 || BBID == 0) {
+ for (auto hotBBInfo : hotBBInfos) {
+ hotBBInfo->HotBBSize++;
+ }
+ }
+ }
+ else {
+ hotBBInfos.clear();
+ SmallVector<StringRef, 2> NamesWithTotalBBSize;
+ s.split(NamesWithTotalBBSize, ' ');
+ if (NamesWithTotalBBSize.size() != 2)
+ error("invalid propeller profile at line: " + Twine(line));
+ unsigned long long TotalBBSize;
+ if (getAsUnsignedInteger(NamesWithTotalBBSize[1], 10, TotalBBSize))
+ error("invalid propeller profile at line: " + Twine(line));
+ SmallVector<StringRef> funcNames;
+ NamesWithTotalBBSize[0].split(funcNames, '/');
+ for (auto funcName : funcNames) {
+ names.push_back({funcName, {static_cast<uint32_t>(TotalBBSize), 0}});
+ hotBBInfos.push_back(&names.back().second);
+ }
+ }
+ }
+ SmallVector<StringRef, 0> symorders;
+ static std::vector<std::string> storedStrings;
+ for (auto item : names) {
+ if (item.second.HotBBSize > 0) {
+ symorders.push_back(item.first);
+ }
+ }
+ for (auto item : names) {
+ if (item.second.HotBBSize != item.second.TotalBBSize) {
+ std::string str(item.first.str());
+ str.append(".cold");
+ storedStrings.push_back(str);
+ symorders.emplace_back(storedStrings.back());
+ }
+ }
+ return symorders;
+}
+
static bool getIsRela(Ctx &ctx, opt::InputArgList &args) {
// The psABI specifies the default relocation entry format.
bool rela = is_contained({EM_AARCH64, EM_AMDGPU, EM_HEXAGON, EM_LOONGARCH,
@@ -1848,6 +1915,16 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
ctx.arg.symbolOrderingFile = getSymbolOrderingFile(ctx, *buffer);
}
+ if (auto *arg = args.getLastArg(OPT_propeller_profile_use)){
+ if (args.hasArg(OPT_call_graph_ordering_file) || args.hasArg(OPT_symbol_ordering_file))
+ error("--propeller-profile-use and --symbol-ordering-file and --call-graph-order-file "
+ "may not be used together");
+ if (auto buffer = readFile(ctx, arg->getValue())) {
+ ctx.arg.ltoPropellerProfile = arg->getValue();
+ ctx.arg.symbolOrderingFile = generateSymbolOrderingFromPropellerProfile(*buffer);
+ }
+ }
+
assert(ctx.arg.versionDefinitions.empty());
ctx.arg.versionDefinitions.push_back(
{"local", (uint16_t)VER_NDX_LOCAL, {}, {}});
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 195526bf390d2..c91b5f7748f8d 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -128,6 +128,7 @@ static lto::Config createConfig(Ctx &ctx) {
c.StatsFile = std::string(ctx.arg.optStatsFilename);
c.SampleProfile = std::string(ctx.arg.ltoSampleProfile);
+ c.PropellerProfile = std::string(ctx.arg.ltoPropellerProfile);
for (StringRef pluginFn : ctx.arg.passPlugins)
c.PassPlugins.push_back(std::string(pluginFn));
c.DebugPassManager = ctx.arg.ltoDebugPassManager;
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 76d28096f82c8..dd4c9d2226fbe 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -408,6 +408,9 @@ def print_map: F<"print-map">,
def print_memory_usage: F<"print-memory-usage">,
HelpText<"Report target memory usage">;
+defm propeller_profile_use:
+ EEq<"propeller-profile-use", "Propeller profile file used to reorder symorder">;
+
defm relax: BB<"relax",
"Enable target-specific relaxations if supported (default)",
"Disable target-specific relaxations">;
diff --git a/llvm/include/llvm/CodeGen/FuncHotBBHashesProfileReader.h b/llvm/include/llvm/CodeGen/FuncHotBBHashesProfileReader.h
new file mode 100644
index 0000000000000..87ac370366f1b
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/FuncHotBBHashesProfileReader.h
@@ -0,0 +1,64 @@
+#ifndef LLVM_CODEGEN_FuncHotBBHashesProfileReader_H
+#define LLVM_CODEGEN_FuncHotBBHashesProfileReader_H
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+
+struct HotBBInfo {
+ uint64_t BBHash;
+ uint64_t Freq;
+};
+
+class FuncHotBBHashesProfileReader : public ImmutablePass {
+public:
+ static char ID;
+
+ FuncHotBBHashesProfileReader(const std::string PropellerProfile);
+
+ FuncHotBBHashesProfileReader();
+
+ StringRef getPassName() const override {
+ return "Basic Block Frequency with Hash Profile Reader";
+ }
+
+ // return a vector of hit BB hashes for a function.
+ std::pair<bool, SmallVector<HotBBInfo, 4>>
+ getHotBBInfosForFunction(StringRef FuncName) const;
+
+ // Reads the profile for matching functions.
+ bool doInitialization(Module &M) override;
+
+private:
+ std::unique_ptr<MemoryBuffer> MBuf;
+
+ StringRef getAliasName(StringRef FuncName) const {
+ auto R = FuncAliasMap.find(FuncName);
+ return R == FuncAliasMap.end() ? FuncName : R->second;
+ }
+
+ // Reads the basic block frequency with hash profile for functions in this module.
+ Error ReadProfile();
+
+ // Profile file path.
+ std::string PropellerFilePath;
+
+ // Some functions have alias names. We use this map to find the main alias
+ // name for which we have mapping in ProgramBBClusterInfo.
+ StringMap<StringRef> FuncAliasMap;
+
+ // record the frequency of basic block,
+ // the basic block is represented by its hash.
+ StringMap<SmallVector<HotBBInfo, 4>> FuncToHotBBHashes;
+};
+
+ImmutablePass *
+createFuncHotBBHashesProfileReaderPass(const std::string PropellerProfile);
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_FuncHotBBHashesProfileReader_H
\ No newline at end of file
diff --git a/llvm/include/llvm/CodeGen/HotMachineBasicBlockInfoGenerator.h b/llvm/include/llvm/CodeGen/HotMachineBasicBlockInfoGenerator.h
new file mode 100644
index 0000000000000..d9f0c21100531
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/HotMachineBasicBlockInfoGenerator.h
@@ -0,0 +1,51 @@
+#ifndef LLVM_CODEGEN_HotMachineBasicBlockInfoGenerator_H
+#define LLVM_CODEGEN_HotMachineBasicBlockInfoGenerator_H
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Transforms/Utils/SampleProfileInference.h"
+#include "llvm/CodeGen/FuncHotBBHashesProfileReader.h"
+
+namespace llvm {
+
+class HotMachineBasicBlockInfoGenerator : public MachineFunctionPass {
+public:
+ static char ID;
+ HotMachineBasicBlockInfoGenerator();
+
+ StringRef getPassName() const override {
+ return "Basic Block Matching and Inference";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+
+ std::optional<SmallVector<MachineBasicBlock *, 4>> getHotMBBs(StringRef FuncName) const;
+
+private:
+ using Edge = std::pair<const MachineBasicBlock *, const MachineBasicBlock *>;
+ using BlockWeightMap = DenseMap<const MachineBasicBlock *, uint64_t>;
+ using EdgeWeightMap = DenseMap<Edge, uint64_t>;
+ using BlockEdgeMap =
+ DenseMap<const MachineBasicBlock *, SmallVector<const MachineBasicBlock *, 8>>;
+
+ DenseMap<StringRef, SmallVector<MachineBasicBlock *, 4>> FuncToHotMBBs;
+
+ void matchHotBBsByHashes(
+ MachineFunction &MF,
+ SmallVector<HotBBInfo, 4> &HotMBBInfos,
+ BlockWeightMap &MBBToFreq,
+ BlockEdgeMap &Successors,
+ SmallVector<MachineBasicBlock *, 4> &HotBBs);
+
+ void generateHotBBsforFunction(
+ MachineFunction &MF,
+ BlockWeightMap &OriBlockWeights,
+ BlockWeightMap &BlockWeights,
+ EdgeWeightMap &EdgeWeights,
+ SmallVector<MachineBasicBlock *, 4> &HotBBs);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_HotMachineBasicBlockInfoGenerator_H
\ No newline at end of file
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index d214ab9306c2f..e8a8378257f2b 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -61,6 +61,10 @@ namespace llvm {
/// This pass garbage-collects such basic blocks.
MachineFunctionPass *createGCEmptyBasicBlocksPass();
+ /// createHotMachineBasicBlockInfoGenerator Pass - This pass generate the hot
+ /// machine basic block clusters that is used in asicBlockSections pass.
+ MachineFunctionPass *createHotMachineBasicBlockInfoGeneratorPass();
+
/// createBasicBlockSections Pass - This pass assigns sections to machine
/// basic blocks and is enabled with -fbasic-block-sections.
MachineFunctionPass *createBasicBlockSectionsPass();
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 38289da1449d5..61eb4bf538f11 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -118,6 +118,7 @@ void initializeFinalizeMachineBundlesPass(PassRegistry &);
void initializeFixIrreduciblePass(PassRegistry &);
void initializeFixupStatepointCallerSavedLegacyPass(PassRegistry &);
void initializeFlattenCFGLegacyPassPass(PassRegistry &);
+void initializeFuncHotBBHashesProfileReaderPass(PassRegistry&);
void initializeFuncletLayoutPass(PassRegistry &);
void initializeGCEmptyBasicBlocksPass(PassRegistry &);
void initializeGCMachineCodeAnalysisPass(PassRegistry &);
@@ -127,6 +128,7 @@ void initializeGlobalMergeFuncPassWrapperPass(PassRegistry &);
void initializeGlobalMergePass(PassRegistry &);
void initializeGlobalsAAWrapperPassPass(PassRegistry &);
void initializeHardwareLoopsLegacyPass(PassRegistry &);
+void initializeHotMachineBasicBlockInfoGeneratorPass(PassRegistry&);
void initializeMIRProfileLoaderPassPass(PassRegistry &);
void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry &);
void initializeIRTranslatorPass(PassRegistry &);
diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h
index a49cce9f30e20..819a76b434ae9 100644
--- a/llvm/include/llvm/LTO/Config.h
+++ b/llvm/include/llvm/LTO/Config.h
@@ -126,6 +126,9 @@ struct Config {
/// Name remapping file for profile data.
std::string ProfileRemapping;
+ /// Name propeller profile.
+ std::string PropellerProfile;
+
/// The directory to store .dwo files.
std::string DwoDir;
diff --git a/llvm/include/llvm/Support/PGOOptions.h b/llvm/include/llvm/Support/PGOOptions.h
index de981abf18705..3af12ee494d06 100644
--- a/llvm/include/llvm/Support/PGOOptions.h
+++ b/llvm/include/llvm/Support/PGOOptions.h
@@ -30,6 +30,7 @@ struct PGOOptions {
enum class ColdFuncOpt { Default, OptSize, MinSize, OptNone };
PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
std::string ProfileRemappingFile, std::string MemoryProfile,
+ std::string PropellerProfile,
IntrusiveRefCntPtr<vfs::FileSystem> FS,
PGOAction Action = NoAction, CSPGOAction CSAction = NoCSAction,
ColdFuncOpt ColdType = ColdFuncOpt::Default,
@@ -44,6 +45,7 @@ struct PGOOptions {
std::string CSProfileGenFile;
std::string ProfileRemappingFile;
std::string MemoryProfile;
+ std::string PropellerProfileFile;
PGOAction Action;
CSPGOAction CSAction;
ColdFuncOpt ColdOptType;
diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp
index b02a7afe771cd..2b9091875f34d 100644
--- a/llvm/lib/CodeGen/BasicBlockSections.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSections.cpp
@@ -70,9 +70,10 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/MachineBlockHashInfo.h"
#include "llvm/CodeGen/BasicBlockSectionUtils.h"
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
+#include "llvm/CodeGen/HotMachineBasicBlockInfoGenerator.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
@@ -135,6 +136,7 @@ INITIALIZE_PASS_BEGIN(
"Prepares for basic block sections, by splitting functions "
"into clusters of basic blocks.",
false, false)
+INITIALIZE_PASS_DEPENDENCY(HotMachineBasicBlockInfoGenerator)
INITIALIZE_PASS_DEPENDENCY(MachineBlockHashInfo)
INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)
INITIALIZE_PASS_END(BasicBlockSections, "bbsections-prepare",
@@ -176,6 +178,30 @@ updateBranches(MachineFunction &MF,
}
}
+// This function generates the machine basic block clusters of "hot" blocks.
+static std::pair<bool, SmallVector<BBClusterInfo>>
+createBBClusterInfoForFunction(
+ const MachineFunction &MF,
+ HotMachineBasicBlockInfoGenerator *HotBBGenerator) {
+ unsigned CurrentCluster = 0;
+ auto OptHotBBs = HotBBGenerator->getHotMBBs(MF.getName());
+ if (!OptHotBBs)
+ return std::pair(false, SmallVector<BBClusterInfo>{});
+ auto& HotMBBs = *OptHotBBs;
+ if(!HotMBBs.empty()) {
+ SmallVector<BBClusterInfo, 4> BBClusterInfos;
+ unsigned CurrentPosition = 0;
+ for (auto &MBB : HotMBBs) {
+ if (MBB->getBBID()) {
+ BBClusterInfos.push_back({*(MBB->getBBID()), CurrentCluster, CurrentPosition++});
+ }
+ }
+ return std::pair(true, std::move(BBClusterInfos));
+ }
+ return std::pair(false, SmallVector<BBClusterInfo>{});
+}
+
+
// This function sorts basic blocks according to the cluster's information.
// All explicitly specified clusters of basic blocks will be ordered
// accordingly. All non-specified BBs go into a separate "Cold" section.
@@ -297,6 +323,11 @@ bool llvm::hasInstrProfHashMismatch(MachineFunction &MF) {
// according to the specification provided by the -fbasic-block-sections flag.
bool BasicBlockSections::handleBBSections(MachineFunction &MF) {
auto BBSectionsType = MF.getTarget().getBBSectionsType();
+ auto PGOOpt = MF.getTarget().getPGOOption();
+ bool HavePropellerProfile = PGOOpt && !PGOOpt->PropellerProfileFile.empty();
+ if (HavePropellerProfile) {
+ BBSectionsType = BasicBlockSection::List;
+ }
if (BBSectionsType == BasicBlockSection::None)
return false;
@@ -306,7 +337,7 @@ bool BasicBlockSections::handleBBSections(MachineFunction &MF) {
// clusters of basic blocks using basic block ids. Source drift can
// invalidate these groupings leading to sub-optimal code generation with
// regards to performance.
- if (BBSectionsType == BasicBlockSection::List &&
+ if (BBSectionsType == BasicBlockSection::List && !HavePropellerProfile &&
hasInstrProfHashMismatch(MF))
return false;
// Renumber blocks before sorting them. This is useful for accessing the
@@ -315,12 +346,16 @@ bool BasicBlockSections::handleBBSections(MachineFunction &MF) {
DenseMap<UniqueBBID, BBClusterInfo> FuncClusterInfo;
if (BBSectionsType == BasicBlockSection::List) {
- auto [HasProfile, ClusterInfo] =
+ std::pair<bool, SmallVector<BBClusterInfo>> ExpClusterInfo =
getAnalysis<BasicBlockSectionsProfileReaderWrapperPass>()
.getClusterInfoForFunction(MF.getName());
- if (!HasProfile)
+ if (!ExpClusterInfo.first) {
+ auto HotBBGenerator = &getAnalysis<HotMachineBasicBlockInfoGenerator>();
+ ExpClusterInfo = createBBClusterInfoForFunction(MF, HotBBGenerator);
+ }
+ if (!ExpClusterInfo.first)
return false;
- for (auto &BBClusterInfo : ClusterInfo) {
+ for (auto &BBClusterInfo : ExpClusterInfo.second) {
FuncClusterInfo.try_emplace(BBClusterInfo.BBID, BBClusterInfo);
}
}
@@ -402,8 +437,9 @@ bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {
void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesAll();
- AU.addRequired<MachineBlockHashInfo>();
AU.addRequired<BasicBlockSectionsProfileReaderWrapperPass>();
+ AU.addRequired<HotMachineBasicBlockInfoGenerator>();
+ AU.addRequired<MachineBlockHashInfo>();
AU.addUsedIfAvailable<MachineDominatorTreeWrapperPass>();
AU.addUsedIfAvailable<MachinePostDominatorTreeWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 424f7b98e2c91..76cd93755005c 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -66,6 +66,7 @@ add_llvm_component_library(LLVMCodeGen
FEntryInserter.cpp
FinalizeISel.cpp
FixupStatepointCallerSaved.cpp
+ FuncHotBBHashesProfileReader.cpp
FuncletLayout.cpp
MachineFunctionAnalysis.cpp
GCMetadata.cpp
@@ -74,6 +75,7 @@ add_llvm_component_library(LLVMCodeGen
GlobalMerge.cpp
GlobalMergeFunctions.cpp
HardwareLoops.cpp
+ HotMachineBasicBlockInfoGenerator.cpp
IfConversion.cpp
ImplicitNullChecks.cpp
IndirectBrExpandPass.cpp
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 0ff5d6006c87a..c30e76366a331 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -47,10 +47,12 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
initializeFinalizeISelPass(Registry);
initializeFinalizeMachineBundlesPass(Registry);
initializeFixupStatepointCallerSavedLegacyPass(Registry);
+ initializeFuncHotBBHashesProfileReaderPass(Registry);
initializeFuncletLayoutPass(Registry);
initializeGCMachineCodeAnalysisPass(Registry);
initializeGCModuleInfoPass(Registry);
initializeHardwareLoopsLegacyPass(Registry);
+ initializeHotMachineBasicBlockInfoGeneratorPass(Registry);
initializeIfConverterPass(Registry);
initializeImplicitNullChecksPass(Registry);
initializeIndirectBrExpandLegacyPassPass(Registry);
diff --git a/llvm/lib/CodeGen/FuncHotBBHashesProfileReader.cpp b/llvm/lib/CodeGen/FuncHotBBHashesProfileReader.cpp
new file mode 100644
index 0000000000000..1bbd046e5a1f0
--- /dev/null
+++ b/llvm/lib/CodeGen/FuncHotBBHashesProfileReader.cpp
@@ -0,0 +1,123 @@
+#include "llvm/CodeGen/FuncHotBBHashesProfileReader.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/LineIterator.h"
+#include <fstream>
+
+using namespace llvm;
+
+char FuncHotBBHashesProfileReader::ID = 0;
+INITIALIZE_PASS(FuncHotBBHashesProfileReader, "func-hotbb-hashes-reader",
+ "Read and parse the hashes of hot basic blocks for function.", false,
+ false)
+
+FuncHotBBHashesProfileReader::FuncHotBBHashesProfileReader(const std::string PropellerProfile)
+ : ImmutablePass(ID), PropellerFilePath(PropellerProfile) {
+ initializeFuncHotBBHashesProfileReaderPass(
+ *PassRegistry::getPassRegistry());
+}
+
+FuncHotBBHashesProfileReader::FuncHotBBHashesProfileReader() : ImmutablePass(ID) {
+ initializeFuncHotBBHashesProfileReaderPass(
+ *PassRegistry::getPassRegistry());
+}
+
+std::pair<bool, SmallVector<HotBBInfo, 4>>
+FuncHotBBHashesProfileReader::getHotBBInfosForFunction(StringRef FuncName) const {
+ auto R = FuncToHotBBHashes.find(getAliasName(FuncName));
+ return R != FuncToHotBBHashes.end()
+ ? std::pair(true, R->second)
+ : std::pair(false, SmallVector<HotBBInfo, 4>{});
+}
+
+// Reads the basic block frequency with hash profile for functions in this module.
+// The profile record the map from basic block hash to basic block frequency of
+// each function. The profile format looks like this:
+// ---------------------------------
+// !foo
+// !!0x123 156 0
+// !!0x456 300 2
+Error FuncHotBBHashesProfileReader::ReadProfile() {
+ ErrorOr<std::unique_ptr<MemoryBuffer>> buffer = MemoryBuffer::getFile(PropellerFilePath);
+ if (!buffer) {
+ return make_error<StringError>(Twine("Invalid propeller profile."), inconvertibleErrorCode());
+ }
+ MBuf = std::move(*buffer);
+ line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#');
+
+ auto invalidProfileError = [&](auto Message) {
+ return make_error<StringError>(
+ Twine("Invalid profile " + MBuf->getBufferIdentifier() + " at line " +
+ Twine(LineIt.line_number()) + ": " + Message),
+ inconvertibleErrorCode());
+ };
+
+ auto FI = FuncToHotBBHashes.end();
+
+ for (; !LineIt.is_at_eof(); ++LineIt) {
+ StringRef S(*LineIt);
+ // Check for the leading "!"
+ if (!S.consume_front("!") || S.empty())
+ break;
+ // Check for second "!" which indicates a basic block hash.
+ if (S.consume_front("!")) {
+ // Skip the profile when we the profile iterator (FI) refers to the
+ // past-the-end element.
+ if (FI == FuncToHotBBHashes.end())
+ continue;
+ SmallVector<StringRef, 3> BBHashes;
+ S.split(BBHashes, ' ');
+ if (BBHashes.size() != 3) {
+ return invalidProfileError("Unexpected elem number.");
+ }
+ unsigned long long Hash, Freq;
+ BBHashes[0].consume_front("0x");
+ if (getAsUnsignedInteger(BBHashes[0], 16, Hash)) {
+ return invalidProfileError(Twine("Unsigned integer expected: '") +
+ BBHashes[0] + "'.");
+ }
+ if (getAsUnsignedInteger(BBHashes[1], 10, Freq)) {
+ return invalidProfileError(Twine("Unsigned integer expected: '") +
+ BBHashes[1] + "'.");
+ }
+ auto It = std::find_if(FI->second.begin(), FI->second.end(),
+ [Hash](HotBBInfo &BBInfo) { return BBInfo.BBHash == Hash; });
+ if (It == FI->second.end())
+ FI->second.push_back({Hash, Freq});
+ } else {
+ // This is a function name specifier.
+ auto [AliasesStr, TotalBBSize] = S.split(' ');
+ // Function aliases are separated using '/'. We use the first function
+ // name for the cluster info mapping and delegate all other aliases to
+ // this one.
+ SmallVector<StringRef, 4> Aliases;
+ AliasesStr.split(Aliases, '/');
+ for (size_t i = 1; i < Aliases.size(); ++i)
+ FuncAliasMap.try_emplace(Aliases[i], Aliases.front());
+
+ // Prepare for parsing clusters of this function name.
+ // Start a new cluster map for this function name.
+ auto R = FuncToHotBBHashes.try_emplace(Aliases.front());
+ // Report error when multiple profiles have been specified for the same
+ // function.
+ if (!R.second)
+ return invalidProfileError("Duplicate profile for function '" +
+ Aliases.front() + "'.");
+ FI = R.first;
+ }
+ }
+ return Error::success();
+}
+
+bool FuncHotBBHashesProfileReader::doInitialization(Module &M) {
+ if (PropellerFilePath.empty())
+ return false;
+ if (auto Err = ReadProfile())
+ report_fatal_error(std::move(Err));
+ return false;
+}
+
+ImmutablePass *
+llvm::createFuncHotBBHashesProfileReaderPass(const std::string PropellerProfile) {
+ return new FuncHotBBHashesProfileReader(PropellerProfile);
+}
\ No newline at end of file
diff --git a/llvm/lib/CodeGen/HotMachineBasicBlockInfoGenerator.cpp b/llvm/lib/CodeGen/HotMachineBasicBlockInfoGenerator.cpp
new file mode 100644
index 0000000000000..66f8fc4e38081
--- /dev/null
+++ b/llvm/lib/CodeGen/HotMachineBasicBlockInfoGenerator.cpp
@@ -0,0 +1,202 @@
+#include "llvm/CodeGen/HotMachineBasicBlockInfoGenerator.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/CodeGen/MachineBlockHashInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Transforms/Utils/CodeLayout.h"
+#include "llvm/Target/TargetMachine.h"
+#include <unordered_set>
+#include <llvm/Support/CommandLine.h>
+
+using namespace llvm;
+
+static cl::opt<bool> PropellerMatchInfer("propeller-match-infer",
+ cl::desc("Use match&infer to evaluate stale profile"), cl::init(false), cl::Optional);
+
+static cl::opt<float> PropellerInferThreshold("propeller-infer-threshold",
+ cl::desc("Threshold for infer stale profile"), cl::init(0.6), cl::Optional);
+
+/// The object is used to identify and match basic blocks given their hashes.
+class StaleMatcher {
+public:
+ /// Initialize stale matcher.
+ void init(const std::vector<MachineBasicBlock *> &Blocks,
+ const std::vector<BlendedBlockHash> &Hashes) {
+ assert(Blocks.size() == Hashes.size() &&
+ "incorrect matcher initialization");
+ for (size_t I = 0; I < Blocks.size(); I++) {
+ MachineBasicBlock *Block = Blocks[I];
+ uint16_t OpHash = Hashes[I].OpcodeHash;
+ OpHashToBlocks[OpHash].push_back(std::make_pair(Hashes[I], Block));
+ }
+ }
+
+ /// Find the most similar block for a given hash.
+ MachineBasicBlock *matchBlock(BlendedBlockHash BlendedHash) const {
+ auto BlockIt = OpHashToBlocks.find(BlendedHash.OpcodeHash);
+ if (BlockIt == OpHashToBlocks.end()) {
+ return nullptr;
+ }
+ MachineBasicBlock *BestBlock = nullptr;
+ uint64_t BestDist = std::numeric_limits<uint64_t>::max();
+ for (auto It : BlockIt->second) {
+ MachineBasicBlock *Block = It.second;
+ BlendedBlockHash Hash = It.first;
+ uint64_t Dist = Hash.distance(BlendedHash);
+ if (BestBlock == nullptr || Dist < BestDist) {
+ BestDist = Dist;
+ BestBlock = Block;
+ }
+ }
+ return BestBlock;
+ }
+
+private:
+ using HashBlockPairType = std::pair<BlendedBlockHash, MachineBasicBlock *>;
+ std::unordered_map<uint16_t, std::vector<HashBlockPairType>> OpHashToBlocks;
+};
+
+INITIALIZE_PASS_BEGIN(HotMachineBasicBlockInfoGenerator, "machine-block-match-infer",
+ "Machine Block Matching and Inference Analysis", true, true)
+INITIALIZE_PASS_DEPENDENCY(MachineBlockHashInfo)
+INITIALIZE_PASS_DEPENDENCY(FuncHotBBHashesProfileReader)
+INITIALIZE_PASS_END(HotMachineBasicBlockInfoGenerator, "machine-block-match-infer",
+ "Machine Block Matching and Inference Analysis", true, true)
+
+char HotMachineBasicBlockInfoGenerator::ID = 0;
+
+HotMachineBasicBlockInfoGenerator::HotMachineBasicBlockInfoGenerator() : MachineFunctionPass(ID) {
+ initializeHotMachineBasicBlockInfoGeneratorPass(*PassRegistry::getPassRegistry());
+}
+
+void HotMachineBasicBlockInfoGenerator::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<MachineBlockHashInfo>();
+ AU.addRequired<FuncHotBBHashesProfileReader>();
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+std::optional<SmallVector<MachineBasicBlock *, 4>>
+HotMachineBasicBlockInfoGenerator::getHotMBBs(StringRef FuncName) const {
+ auto It = FuncToHotMBBs.find(FuncName);
+ if (It == FuncToHotMBBs.end()) {
+ return std::nullopt;
+ }
+ return It->second;
+}
+
+void HotMachineBasicBlockInfoGenerator::matchHotBBsByHashes(
+ MachineFunction &MF,
+ SmallVector<HotBBInfo, 4> &HotMBBInfos,
+ BlockWeightMap &MBBToFreq,
+ BlockEdgeMap &Successors,
+ SmallVector<MachineBasicBlock *, 4> &HotBBs) {
+ std::vector<MachineBasicBlock *> Blocks;
+ std::vector<BlendedBlockHash> Hashes;
+ auto MBHI = &getAnalysis<MachineBlockHashInfo>();
+ for (auto &Block : MF) {
+ Blocks.push_back(&Block);
+ Hashes.push_back(BlendedBlockHash(MBHI->getMBBHash(Block)));
+ for (auto *Succ : Block.successors()) {
+ Successors[&Block].push_back(Succ);
+ }
+ }
+ StaleMatcher Matcher;
+ Matcher.init(Blocks, Hashes);
+ for (auto &item : HotMBBInfos) {
+ MachineBasicBlock *Block
+ = Matcher.matchBlock(BlendedBlockHash(item.BBHash));
+ if (Block != nullptr) {
+ HotBBs.push_back(Block);
+ MBBToFreq[Block] = item.Freq;
+ }
+ }
+}
+
+void HotMachineBasicBlockInfoGenerator::generateHotBBsforFunction(
+ MachineFunction &MF,
+ BlockWeightMap &OriBlockWeights,
+ BlockWeightMap &BlockWeights,
+ EdgeWeightMap &EdgeWeights,
+ SmallVector<MachineBasicBlock *, 4> &HotBBs) {
+ if (!PropellerMatchInfer) {
+ for (auto MBB : HotBBs) {
+ if (MBB->isEntryBlock() || OriBlockWeights[MBB] > 0) {
+ FuncToHotMBBs[MF.getName()].push_back(MBB);
+ }
+ }
+ return;
+ }
+
+ if (MF.size() <= 2) {
+ for (auto &MBB : MF) {
+ if (MBB.isEntryBlock() || BlockWeights[&MBB] > 0) {
+ FuncToHotMBBs[MF.getName()].push_back(&MBB);
+ }
+ }
+ return;
+ }
+
+ MF.RenumberBlocks();
+
+ SmallVector<uint64_t, 0> BlockSizes(MF.size());
+ SmallVector<uint64_t, 0> BlockCounts(MF.size());
+ std::vector<MachineBasicBlock *> OrigOrder;
+ OrigOrder.reserve(MF.size());
+ SmallVector<codelayout::EdgeCount, 0> JumpCounts;
+
+ // Init the MBB size and count.
+ for (auto &MBB : MF) {
+ auto NonDbgInsts =
+ instructionsWithoutDebug(MBB.instr_begin(), MBB.instr_end());
+ int NumInsts = std::distance(NonDbgInsts.begin(), NonDbgInsts.end());
+ BlockSizes[MBB.getNumber()] = 4 * NumInsts;
+ BlockCounts[MBB.getNumber()] = BlockWeights[&MBB];
+ OrigOrder.push_back(&MBB);
+ }
+
+ // Init the edge count.
+ for (auto &MBB : MF) {
+ for (auto *Succ : MBB.successors()) {
+ auto EdgeWeight = EdgeWeights[std::make_pair(&MBB, Succ)];
+ JumpCounts.push_back({static_cast<uint64_t>(MBB.getNumber()), static_cast<uint64_t>(Succ->getNumber()), EdgeWeight});
+ }
+ }
+
+ // Run the layout algorithm
+ auto Result = computeExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
+ for (uint64_t R : Result) {
+ auto Block = OrigOrder[R];
+ if (Block->isEntryBlock() || BlockWeights[Block] > 0)
+ FuncToHotMBBs[MF.getName()].push_back(Block);
+ }
+}
+
+bool HotMachineBasicBlockInfoGenerator::runOnMachineFunction(MachineFunction &MF) {
+ auto [FindFlag, HotMBBInfos] =
+ getAnalysis<FuncHotBBHashesProfileReader>()
+ .getHotBBInfosForFunction(MF.getName());
+ if (!FindFlag) {
+ return false;
+ }
+ BlockWeightMap MBBToFreq;
+ BlockEdgeMap Successors;
+ SmallVector<MachineBasicBlock *, 4> HotBBs;
+ matchHotBBsByHashes(MF, HotMBBInfos, MBBToFreq, Successors, HotBBs);
+
+ // If the ratio of the number of MBBs in matching to the total number of MBBs in the
+ // function is less than the threshold value, the processing should be abandoned.
+ if (static_cast<float>(HotBBs.size()) / MF.size() < PropellerInferThreshold) {
+ return false;
+ }
+
+ SampleProfileInference<MachineFunction> SPI(MF, Successors, MBBToFreq);
+ BlockWeightMap BlockWeights;
+ EdgeWeightMap EdgeWeights;
+ SPI.apply(BlockWeights, EdgeWeights);
+ generateHotBBsforFunction(MF, MBBToFreq, BlockWeights, EdgeWeights, HotBBs);
+ return false;
+}
+
+MachineFunctionPass *llvm::createHotMachineBasicBlockInfoGeneratorPass() {
+ return new HotMachineBasicBlockInfoGenerator();
+}
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 1bf6621cdf791..7a097bd81013f 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -23,6 +23,7 @@
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
#include "llvm/CodeGen/CSEConfigBase.h"
#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
+#include "llvm/CodeGen/FuncHotBBHashesProfileReader.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachinePassRegistry.h"
#include "llvm/CodeGen/Passes.h"
@@ -1270,15 +1271,25 @@ void TargetPassConfig::addMachinePasses() {
addPass(createStaticDataSplitterPass());
addPass(createStaticDataAnnotatorPass());
}
+
+ auto PGOOpt = TM->getPGOOption();
+ bool HavePropellerProfile = PGOOpt && !PGOOpt->PropellerProfileFile.empty();
+ if (HavePropellerProfile) {
+ addPass(llvm::createFuncHotBBHashesProfileReaderPass((*PGOOpt).PropellerProfileFile));
+ }
+
// We run the BasicBlockSections pass if either we need BB sections or BB
// address map (or both).
if (TM->getBBSectionsType() != llvm::BasicBlockSection::None ||
- TM->Options.BBAddrMap) {
+ TM->Options.BBAddrMap || HavePropellerProfile) {
if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) {
addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass(
TM->getBBSectionsFuncListBuf()));
addPass(llvm::createBasicBlockPathCloningPass());
}
+ if (HavePropellerProfile) {
+ addPass(llvm::createHotMachineBasicBlockInfoGeneratorPass());
+ }
addPass(llvm::createBasicBlockSectionsPass());
}
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 1c764a0188eda..694210c179258 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -246,22 +246,28 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
std::optional<PGOOptions> PGOOpt;
if (!Conf.SampleProfile.empty())
PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping,
- /*MemoryProfile=*/"", FS, PGOOptions::SampleUse,
- PGOOptions::NoCSAction,
+ /*MemoryProfile=*/"", Conf.PropellerProfile,
+ FS, PGOOptions::SampleUse, PGOOptions::NoCSAction,
PGOOptions::ColdFuncOpt::Default, true);
else if (Conf.RunCSIRInstr) {
PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping,
- /*MemoryProfile=*/"", FS, PGOOptions::IRUse,
- PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default,
- Conf.AddFSDiscriminator);
+ /*MemoryProfile=*/"", Conf.PropellerProfile,
+ FS, PGOOptions::IRUse, PGOOptions::CSIRInstr,
+ PGOOptions::ColdFuncOpt::Default, Conf.AddFSDiscriminator);
} else if (!Conf.CSIRProfile.empty()) {
PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping,
- /*MemoryProfile=*/"", FS, PGOOptions::IRUse,
- PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default,
- Conf.AddFSDiscriminator);
+ /*MemoryProfile=*/"", Conf.PropellerProfile,
+ FS, PGOOptions::IRUse, PGOOptions::CSIRUse,
+ PGOOptions::ColdFuncOpt::Default, Conf.AddFSDiscriminator);
NoPGOWarnMismatch = !Conf.PGOWarnMismatch;
+ } else if (!Conf.PropellerProfile.empty()) {
+ PGOOpt = PGOOptions("", "", Conf.ProfileRemapping,
+ /*MemoryProfile=*/"", Conf.PropellerProfile,
+ FS, PGOOptions::NoAction, PGOOptions::NoCSAction,
+ PGOOptions::ColdFuncOpt::Default, Conf.AddFSDiscriminator);
} else if (Conf.AddFSDiscriminator) {
- PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
+ PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"",
+ /*PropellerProfile=*/"", nullptr,
PGOOptions::NoAction, PGOOptions::NoCSAction,
PGOOptions::ColdFuncOpt::Default, true);
}
diff --git a/llvm/lib/Support/PGOOptions.cpp b/llvm/lib/Support/PGOOptions.cpp
index 5981dff9e0946..040b1b18eff63 100644
--- a/llvm/lib/Support/PGOOptions.cpp
+++ b/llvm/lib/Support/PGOOptions.cpp
@@ -14,13 +14,15 @@ using namespace llvm;
PGOOptions::PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
std::string ProfileRemappingFile,
std::string MemoryProfile,
+ std::string PropellerProfile,
IntrusiveRefCntPtr<vfs::FileSystem> FS, PGOAction Action,
CSPGOAction CSAction, ColdFuncOpt ColdType,
bool DebugInfoForProfiling, bool PseudoProbeForProfiling,
bool AtomicCounterUpdate)
: ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile),
ProfileRemappingFile(ProfileRemappingFile), MemoryProfile(MemoryProfile),
- Action(Action), CSAction(CSAction), ColdOptType(ColdType),
+ PropellerProfileFile(PropellerProfile), Action(Action),
+ CSAction(CSAction), ColdOptType(ColdType),
DebugInfoForProfiling(DebugInfoForProfiling ||
(Action == SampleUse && !PseudoProbeForProfiling)),
PseudoProbeForProfiling(PseudoProbeForProfiling),
@@ -45,12 +47,12 @@ PGOOptions::PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
// If neither Action nor CSAction nor MemoryProfile are set,
// DebugInfoForProfiling or PseudoProbeForProfiling needs to be true.
assert(this->Action != NoAction || this->CSAction != NoCSAction ||
- !this->MemoryProfile.empty() || this->DebugInfoForProfiling ||
- this->PseudoProbeForProfiling);
+ !this->MemoryProfile.empty() || !this->PropellerProfileFile.empty() ||
+ this->DebugInfoForProfiling || this->PseudoProbeForProfiling);
// If we need to use the profile, the VFS cannot be nullptr.
assert(this->FS || !(this->Action == IRUse || this->CSAction == CSIRUse ||
- !this->MemoryProfile.empty()));
+ !this->MemoryProfile.empty() || !this->PropellerProfileFile.empty()));
}
PGOOptions::PGOOptions(const PGOOptions &) = default;
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index 730858918e27a..9af649eaf38d6 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -187,6 +187,10 @@ static cl::opt<std::string>
MemoryProfileFile("memory-profile-file",
cl::desc("Path to the memory profile."), cl::Hidden);
+static cl::opt<std::string>
+ PropellerProfileFile("propeller-profile-file",
+ cl::desc("Path to the propeller profile."), cl::Hidden);
+
static cl::opt<CSPGOKind> CSPGOKindFlag(
"cspgo-kind", cl::init(NoCSPGO), cl::Hidden,
cl::desc("The kind of context sensitive profile guided optimization"),
@@ -361,23 +365,26 @@ bool llvm::runPassPipeline(
std::optional<PGOOptions> P;
switch (PGOKindFlag) {
case InstrGen:
- P = PGOOptions(ProfileFile, "", "", MemoryProfileFile, FS,
+ P = PGOOptions(ProfileFile, "", "", MemoryProfileFile,
+ PropellerProfileFile, FS,
PGOOptions::IRInstr, PGOOptions::NoCSAction,
PGOColdFuncAttr);
break;
case InstrUse:
- P = PGOOptions(ProfileFile, "", ProfileRemappingFile, MemoryProfileFile, FS,
+ P = PGOOptions(ProfileFile, "", ProfileRemappingFile,
+ MemoryProfileFile, PropellerProfileFile, FS,
PGOOptions::IRUse, PGOOptions::NoCSAction, PGOColdFuncAttr);
break;
case SampleUse:
- P = PGOOptions(ProfileFile, "", ProfileRemappingFile, MemoryProfileFile, FS,
- PGOOptions::SampleUse, PGOOptions::NoCSAction,
- PGOColdFuncAttr);
+ P = PGOOptions(ProfileFile, "", ProfileRemappingFile,
+ MemoryProfileFile, PropellerProfileFile, FS,
+ PGOOptions::SampleUse, PGOOptions::NoCSAction, PGOColdFuncAttr);
break;
case NoPGO:
if (DebugInfoForProfiling || PseudoProbeForProfiling ||
!MemoryProfileFile.empty())
- P = PGOOptions("", "", "", MemoryProfileFile, FS, PGOOptions::NoAction,
+ P = PGOOptions("", "", "", MemoryProfileFile,
+ PropellerProfileFile, FS, PGOOptions::NoAction,
PGOOptions::NoCSAction, PGOColdFuncAttr,
DebugInfoForProfiling, PseudoProbeForProfiling);
else
@@ -399,8 +406,8 @@ bool llvm::runPassPipeline(
P->CSProfileGenFile = CSProfileGenFile;
} else
P = PGOOptions("", CSProfileGenFile, ProfileRemappingFile,
- /*MemoryProfile=*/"", FS, PGOOptions::NoAction,
- PGOOptions::CSIRInstr);
+ /*MemoryProfile=*/"", /*PropellerProfile=*/"",
+ FS, PGOOptions::NoAction, PGOOptions::CSIRInstr);
} else /* CSPGOKindFlag == CSInstrUse */ {
if (!P) {
errs() << "CSInstrUse needs to be together with InstrUse";
More information about the llvm-commits
mailing list