[llvm] Use the Propeller CFG profile in the PGO analysis map if it is available. (PR #163252)
Rahman Lavaee via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 11 11:03:07 PST 2025
https://github.com/rlavaee updated https://github.com/llvm/llvm-project/pull/163252
>From 1ed10b00056ac04fdc9933966bc0a3525b54a027 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Mon, 13 Oct 2025 17:29:55 +0000
Subject: [PATCH 01/15] Emit the PGO analysis map based on the Propeller
profile.
---
.../CodeGen/BasicBlockSectionsProfileReader.h | 46 ++++++--
llvm/include/llvm/MC/MCContext.h | 2 +-
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 39 +++++--
.../BasicBlockSectionsProfileReader.cpp | 4 +-
.../basic-block-address-map-empty-function.ll | 2 +-
...sic-block-address-map-function-sections.ll | 6 +-
.../basic-block-address-map-pgo-features.ll | 2 +-
...k-address-map-with-basic-block-sections.ll | 2 +-
...sic-block-address-map-with-emit-bb-hash.ll | 2 +-
.../X86/basic-block-address-map-with-mfs.ll | 2 +-
.../CodeGen/X86/basic-block-address-map.ll | 2 +-
.../X86/basic-block-sections-pgo-features.ll | 100 ++++++++++++++++++
12 files changed, 181 insertions(+), 28 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index f0d28d863282e..9ace90fa61394 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -42,18 +42,40 @@ struct BBClusterInfo {
unsigned PositionInCluster;
};
+// This represents the CFG profile data for a function.
+struct CfgProfile {
+ // Node counts for each basic block.
+ DenseMap<UniqueBBID, uint64_t> NodeCounts;
+ // Edge counts for each edge, stored as a nested map.
+ DenseMap<UniqueBBID, DenseMap<UniqueBBID, uint64_t>> EdgeCounts;
+
+ // Returns the profile count for the given basic block or zero if it does not
+ // exist.
+ uint64_t getBlockCount(const UniqueBBID &BBID) const {
+ return NodeCounts.lookup(BBID);
+ }
+
+ // Returns the profile count for the edge from `SrcBBID` to `SinkBBID` or
+ // zero if it does not exist.
+ uint64_t getEdgeCount(const UniqueBBID &SrcBBID,
+ const UniqueBBID &SinkBBID) const {
+ auto It = EdgeCounts.find(SrcBBID);
+ if (It == EdgeCounts.end())
+ return 0;
+ return It->second.lookup(SinkBBID);
+ }
+};
+
// This represents the raw input profile for one function.
-struct FunctionPathAndClusterInfo {
+struct FunctionProfile {
// BB Cluster information specified by `UniqueBBID`s.
SmallVector<BBClusterInfo> ClusterInfo;
// Paths to clone. A path a -> b -> c -> d implies cloning b, c, and d along
// the edge a -> b (a is not cloned). The index of the path in this vector
// determines the `UniqueBBID::CloneID` of the cloned blocks in that path.
SmallVector<SmallVector<unsigned>> ClonePaths;
- // Node counts for each basic block.
- DenseMap<UniqueBBID, uint64_t> NodeCounts;
- // Edge counts for each edge, stored as a nested map.
- DenseMap<UniqueBBID, DenseMap<UniqueBBID, uint64_t>> EdgeCounts;
+ // Cfg profile data (block and edge frequencies).
+ CfgProfile Cfg;
// Hash for each basic block. The Hashes are stored for every original block
// (not cloned blocks), hence the map key being unsigned instead of
// UniqueBBID.
@@ -81,10 +103,14 @@ class BasicBlockSectionsProfileReader {
SmallVector<SmallVector<unsigned>>
getClonePathsForFunction(StringRef FuncName) const;
- // Returns the profile count for the edge from `SrcBBID` to `SinkBBID` in
- // function `FuncName` or zero if it does not exist.
- uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID,
- const UniqueBBID &SinkBBID) const;
+ // Returns a pointer to the CfgProfile for the given function.
+ // Returns nullptr if no profile data is available for the function.
+ const CfgProfile *getFunctionCfgProfile(StringRef FuncName) const {
+ auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
+ if (It == ProgramPathAndClusterInfo.end())
+ return nullptr;
+ return &It->second.Cfg;
+ }
// Return the complete function path and cluster info for the given function.
std::pair<bool, FunctionPathAndClusterInfo>
@@ -136,7 +162,7 @@ class BasicBlockSectionsProfileReader {
// for (all or some of) its basic blocks. The cluster information for every
// basic block includes its cluster ID along with the position of the basic
// block in that cluster.
- StringMap<FunctionPathAndClusterInfo> ProgramPathAndClusterInfo;
+ StringMap<FunctionProfile> ProgramPathAndClusterInfo;
// Some functions have alias names. We use this map to find the main alias
// name which appears in ProgramPathAndClusterInfo as a key.
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 74abe3403bbed..c5de76bd30b3f 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -175,7 +175,7 @@ class MCContext {
unsigned GetInstance(unsigned LocalLabelVal);
/// SHT_LLVM_BB_ADDR_MAP version to emit.
- uint8_t BBAddrMapVersion = 4;
+ uint8_t BBAddrMapVersion = 5;
/// The file name of the log file from the environment variable
/// AS_SECURE_LOG_FILE. Which must be set before the .secure_log_unique
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index c459b80baf4ab..c76b8c3e74bec 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -20,7 +20,6 @@
#include "WinException.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
-#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
@@ -37,6 +36,7 @@
#include "llvm/BinaryFormat/COFF.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/GCMetadataPrinter.h"
#include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h"
@@ -150,6 +150,7 @@ enum class PGOMapFeaturesEnum {
FuncEntryCount,
BBFreq,
BrProb,
+ PropellerCFG,
All,
};
static cl::bits<PGOMapFeaturesEnum> PgoAnalysisMapFeatures(
@@ -166,6 +167,12 @@ static cl::bits<PGOMapFeaturesEnum> PgoAnalysisMapFeatures(
"Enable extended information within the SHT_LLVM_BB_ADDR_MAP that is "
"extracted from PGO related analysis."));
+static cl::opt<bool> PgoAnalysisMapEmitBBSectionsCfg(
+ "pgo-analysis-map-emit-bb-sections-cfg",
+ cl::desc("Enable the post-link cfg information from the basic block "
+ "sections profile in the PGO analysis map"),
+ cl::Hidden, cl::init(false));
+
static cl::opt<bool> BBAddrMapSkipEmitBBEntries(
"basic-block-address-map-skip-bb-entries",
cl::desc("Skip emitting basic block entries in the SHT_LLVM_BB_ADDR_MAP "
@@ -479,6 +486,7 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
if (EmitBBHash)
AU.addRequired<MachineBlockHashInfo>();
+ AU.addUsedIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();
}
bool AsmPrinter::doInitialization(Module &M) {
@@ -1411,7 +1419,7 @@ static uint32_t getBBAddrMapMetadata(const MachineBasicBlock &MBB) {
static llvm::object::BBAddrMap::Features
getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
- bool HasCalls) {
+ bool HasCalls, const CfgProfile *FuncCfgProfile) {
// Ensure that the user has not passed in additional options while also
// specifying all or none.
if ((PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::None) ||
@@ -1433,17 +1441,17 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
bool BrProbEnabled =
AllFeatures ||
(!NoFeatures && PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::BrProb));
+ bool PostLinkCfgEnabled = FuncCfgProfile && PgoAnalysisMapEmitBBSectionsCfg;
if ((BBFreqEnabled || BrProbEnabled) && BBAddrMapSkipEmitBBEntries) {
MF.getFunction().getContext().emitError(
- "BB entries info is required for BBFreq and BrProb "
- "features");
+ "BB entries info is required for BBFreq and BrProb features");
}
return {FuncEntryCountEnabled, BBFreqEnabled, BrProbEnabled,
MF.hasBBSections() && NumMBBSectionRanges > 1,
// Use static_cast to avoid breakage of tests on windows.
static_cast<bool>(BBAddrMapSkipEmitBBEntries), HasCalls,
- static_cast<bool>(EmitBBHash), false};
+ static_cast<bool>(EmitBBHash), PostLinkCfgEnabled};
}
void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
@@ -1452,6 +1460,14 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
assert(BBAddrMapSection && ".llvm_bb_addr_map section is not initialized.");
bool HasCalls = !CurrentFnCallsiteEndSymbols.empty();
+ const BasicBlockSectionsProfileReader *BBSPR = nullptr;
+ if (auto *BBSPRPass =
+ getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>())
+ BBSPR = &BBSPRPass->getBBSPR();
+ const CfgProfile *FuncCfgProfile = nullptr;
+ if (BBSPR)
+ FuncCfgProfile = BBSPR->getFunctionCfgProfile(MF.getFunction().getName());
+
const MCSymbol *FunctionSymbol = getFunctionBegin();
OutStreamer->pushSection();
@@ -1460,7 +1476,8 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
uint8_t BBAddrMapVersion = OutStreamer->getContext().getBBAddrMapVersion();
OutStreamer->emitInt8(BBAddrMapVersion);
OutStreamer->AddComment("feature");
- auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size(), HasCalls);
+ auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size(), HasCalls,
+ FuncCfgProfile);
OutStreamer->emitInt8(Features.encode());
// Emit BB Information for each basic block in the function.
if (Features.MultiBBRange) {
@@ -1565,6 +1582,11 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
OutStreamer->AddComment("basic block frequency");
OutStreamer->emitULEB128IntValue(
MBFI->getBlockFreq(&MBB).getFrequency());
+ if (Features.PostLinkCfg) {
+ OutStreamer->AddComment("basic block frequency (propeller)");
+ OutStreamer->emitULEB128IntValue(
+ FuncCfgProfile->getBlockCount(*MBB.getBBID()));
+ }
}
if (Features.BrProb) {
unsigned SuccCount = MBB.succ_size();
@@ -1576,6 +1598,11 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
OutStreamer->AddComment("successor branch probability");
OutStreamer->emitULEB128IntValue(
MBPI->getEdgeProbability(&MBB, SuccMBB).getNumerator());
+ if (Features.PostLinkCfg) {
+ OutStreamer->AddComment("successor branch frequency (propeller)");
+ OutStreamer->emitULEB128IntValue(FuncCfgProfile->getEdgeCount(
+ *MBB.getBBID(), *SuccMBB->getBBID()));
+ }
}
}
}
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index be1c60c57ccf4..8baa478c7c5c0 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -290,10 +290,10 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
Twine("unsigned integer expected: '") + CountStr + "'");
if (i == 0) {
// The first element represents the source and its total count.
- FI->second.NodeCounts[SrcBBID = *BBID] = Count;
+ FI->second.Cfg.NodeCounts[SrcBBID = *BBID] = Count;
continue;
}
- FI->second.EdgeCounts[SrcBBID][*BBID] = Count;
+ FI->second.Cfg.EdgeCounts[SrcBBID][*BBID] = Count;
}
}
continue;
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll b/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
index 423e31868bc5f..7457e3bdb24dc 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
@@ -19,7 +19,7 @@ entry:
; CHECK: func:
; CHECK: .Lfunc_begin1:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text{{$}}
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; BASIC-NEXT: .byte 0 # feature
; PGO-NEXT: .byte 3 # feature
; CHECK-NEXT: .quad .Lfunc_begin1 # function address
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll b/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
index e32e5222bf016..bdfe1e4564fc1 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
@@ -10,7 +10,7 @@ define dso_local i32 @_Z3barv() {
; CHECK-LABEL: _Z3barv:
; CHECK-NEXT: [[BAR_BEGIN:.Lfunc_begin[0-9]+]]:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3barv{{$}}
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; CHECK-NEXT: .byte 0 # feature
; CHECK-NEXT: .quad [[BAR_BEGIN]] # function address
@@ -23,7 +23,7 @@ define dso_local i32 @_Z3foov() {
; CHECK-LABEL: _Z3foov:
; CHECK-NEXT: [[FOO_BEGIN:.Lfunc_begin[0-9]+]]:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3foov{{$}}
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; CHECK-NEXT: .byte 32 # feature
; CHECK-NEXT: .quad [[FOO_BEGIN]] # function address
@@ -36,6 +36,6 @@ define linkonce_odr dso_local i32 @_Z4fooTIiET_v() comdat {
; CHECK-LABEL: _Z4fooTIiET_v:
; CHECK-NEXT: [[FOOCOMDAT_BEGIN:.Lfunc_begin[0-9]+]]:
; CHECK: .section .llvm_bb_addr_map,"oG", at llvm_bb_addr_map,.text._Z4fooTIiET_v,_Z4fooTIiET_v,comdat{{$}}
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; CHECK-NEXT: .byte 0 # feature
; CHECK-NEXT: .quad [[FOOCOMDAT_BEGIN]] # function address
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
index 12b1297ba97ce..ca3c8d18e7dc5 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
@@ -69,7 +69,7 @@ declare i32 @__gxx_personality_v0(...)
; CHECK-LABEL: .Lfunc_end0:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3bazb{{$}}
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; BASIC-NEXT: .byte 32 # feature
; PGO-ALL-NEXT: .byte 39 # feature
; FEC-ONLY-NEXT:.byte 33 # feature
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
index aeb6dc95e32f3..297cbd81ed036 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
@@ -47,7 +47,7 @@ declare i32 @__gxx_personality_v0(...)
; CHECK-LABEL: .Lfunc_end0:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.hot._Z3bazb
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; CHECK-NEXT: .byte 40 # feature
; CHECK-NEXT: .byte 2 # number of basic block ranges
; CHECK-NEXT: .quad .Lfunc_begin0 # base address
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll
index a567887753245..f17e9a5849150 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll
@@ -50,7 +50,7 @@ declare i32 @__gxx_personality_v0(...)
; UNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3bazb{{$}}
;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
; NOUNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text,unique,1
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; CHECK-NEXT: .byte 96 # feature
; CHECK-NEXT: .quad .Lfunc_begin0 # function address
; CHECK-NEXT: .byte 6 # number of basic blocks
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
index d49b313679628..36e21b9b3faa7 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
@@ -58,7 +58,7 @@ declare i32 @qux()
; CHECK-LABEL: .Lfunc_end0:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.hot.foo
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; BASIC-NEXT: .byte 40 # feature
; PGO-NEXT: .byte 47 # feature
; CHECK-NEXT: .byte 2 # number of basic block ranges
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map.ll b/llvm/test/CodeGen/X86/basic-block-address-map.ll
index 64cf2c709f3df..523b939765525 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map.ll
@@ -52,7 +52,7 @@ declare i32 @__gxx_personality_v0(...)
; UNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3bazb{{$}}
;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
; NOUNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text,unique,1
-; CHECK-NEXT: .byte 4 # version
+; CHECK-NEXT: .byte 5 # version
; CHECK-NEXT: .byte 32 # feature
; CHECK-NEXT: .quad .Lfunc_begin0 # function address
; CHECK-NEXT: .byte 6 # number of basic blocks
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll
new file mode 100644
index 0000000000000..7f7d87369701b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll
@@ -0,0 +1,100 @@
+; Verify PGO analysis map features with basic block sections profile.
+;
+; RUN: echo 'v1' > %t
+; RUN: echo 'f foo' >> %t
+; RUN: echo 'g 0:1000,1:800,2:200 1:800,3:800 2:200,3:200 3:1000' >> %t
+; RUN: echo 'c 0 1 2' >> %t
+;
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t -basic-block-address-map -pgo-analysis-map=all -pgo-analysis-map-emit-bb-sections-cfg | FileCheck %s
+
+define void @foo(i1 %cond) nounwind !prof !0 {
+entry:
+ br label %bb1
+
+bb1:
+ br i1 %cond, label %bb2, label %bb3, !prof !1
+
+bb2:
+ br label %bb3
+
+bb3:
+ ret void
+}
+
+!0 = !{!"function_entry_count", i64 1500}
+!1 = !{!"branch_weights", i32 1200, i32 300}
+
+;; Verify that foo's PGO map contains both IRPGO and Propeller CFG profiles.
+
+; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.foo
+; CHECK-NEXT: .byte 5 # version
+; CHECK-NEXT: .byte 143 # feature
+; CHECK: .quad .Lfunc_begin0 # base address
+; CHECK: .byte 0 # BB id
+; CHECK: .byte 1 # BB id
+; CHECK: .byte 2 # BB id
+; CHECK: .byte 3 # BB id
+
+; CHECK: .ascii "\334\013" # function entry count
+; CHECK-NEXT: .ascii "\200\200\200\200\200\200\200 " # basic block frequency
+; CHECK-NEXT: .ascii "\350\007" # basic block frequency (propeller)
+; CHECK-NEXT: .byte 1 # basic block successor count
+; CHECK-NEXT: .byte 1 # successor BB ID
+; CHECK-NEXT: .ascii "\200\200\200\200\b" # successor branch probability
+; CHECK-NEXT: .ascii "\240\006" # successor branch frequency (propeller)
+; CHECK-NEXT: .ascii "\200\200\200\200\200\200\200 " # basic block frequency
+; CHECK-NEXT: .ascii "\240\006" # basic block frequency (propeller)
+; CHECK-NEXT: .byte 2 # basic block successor count
+; CHECK-NEXT: .byte 2 # successor BB ID
+; CHECK-NEXT: .ascii "\200\200\200\200\004" # successor branch probability
+; CHECK-NEXT: .byte 0 # successor branch frequency (propeller)
+; CHECK-NEXT: .byte 3 # successor BB ID
+; CHECK-NEXT: .ascii "\200\200\200\200\004" # successor branch probability
+; CHECK-NEXT: .ascii "\240\006" # successor branch frequency (propeller)
+; CHECK-NEXT: .ascii "\200\200\200\200\200\200\200\020" # basic block frequency
+; CHECK-NEXT: .ascii "\310\001" # basic block frequency (propeller)
+; CHECK-NEXT: .byte 1 # basic block successor count
+; CHECK-NEXT: .byte 3 # successor BB ID
+; CHECK-NEXT: .ascii "\200\200\200\200\b" # successor branch probability
+; CHECK-NEXT: .ascii "\310\001" # successor branch frequency (propeller)
+; CHECK-NEXT: .ascii "\200\200\200\200\200\200\200 " # basic block frequency
+; CHECK-NEXT: .ascii "\350\007" # basic block frequency (propeller)
+; CHECK-NEXT: .byte 0 # basic block successor count
+
+define void @bar(i1 %cond) nounwind !prof !2 {
+entry:
+ br i1 %cond, label %bb1, label %bb2, !prof !3
+
+bb1:
+ ret void
+
+bb2:
+ ret void
+}
+
+!2 = !{!"function_entry_count", i64 80}
+!3 = !{!"branch_weights", i32 2, i32 78}
+
+;; Verify that the PGO map for bar only includes IRPGO data since it doesn't
+;; have Propeller profile.
+
+; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.bar
+; CHECK-NEXT: .byte 5 # version
+; CHECK-NEXT: .byte 7 # feature
+; CHECK: .quad .Lfunc_begin1 # function address
+; CHECK: .byte 0 # BB id
+; CHECK: .byte 1 # BB id
+; CHECK: .byte 2 # BB id
+
+; CHECK: .byte 80 # function entry count
+; CHECK-NEXT: .ascii "\200\200\200\200\200\200\200 " # basic block frequency
+; CHECK-NEXT: .byte 2 # basic block successor count
+; CHECK-NEXT: .byte 1 # successor BB ID
+; CHECK-NEXT: .ascii "\200\200\200\200\004" # successor branch probability
+; CHECK-NEXT: .byte 2 # successor BB ID
+; CHECK-NEXT: .ascii "\200\200\200\200\004" # successor branch probability
+; CHECK-NEXT: .ascii "\200\200\200\200\200\200\200\020" # basic block frequency
+; CHECK-NEXT: .byte 0 # basic block successor count
+; CHECK-NEXT: .ascii "\200\200\200\200\200\200\200\020" # basic block frequency
+; CHECK-NEXT: .byte 0 # basic block successor count
+
>From a7bcde5aa7ff3260e15a0c8e3b74edd2d59d188f Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 04:24:40 +0000
Subject: [PATCH 02/15] Rename functions in BasicBlockSectionsProfileReader.
---
.../CodeGen/BasicBlockSectionsProfileReader.h | 40 ++++++------
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 14 ++--
.../BasicBlockMatchingAndInference.cpp | 21 +++---
.../BasicBlockSectionsProfileReader.cpp | 64 ++++++++-----------
4 files changed, 63 insertions(+), 76 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 9ace90fa61394..898b36ae035d1 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -43,12 +43,18 @@ struct BBClusterInfo {
};
// This represents the CFG profile data for a function.
-struct CfgProfile {
+struct CFGProfile {
// Node counts for each basic block.
DenseMap<UniqueBBID, uint64_t> NodeCounts;
// Edge counts for each edge, stored as a nested map.
DenseMap<UniqueBBID, DenseMap<UniqueBBID, uint64_t>> EdgeCounts;
+ // Hash for each basic block. The Hashes are stored for every original block
+ // (not cloned blocks), hence the map key being unsigned instead of
+ // UniqueBBID.
+ DenseMap<unsigned, uint64_t> BBHashes;
+
+
// Returns the profile count for the given basic block or zero if it does not
// exist.
uint64_t getBlockCount(const UniqueBBID &BBID) const {
@@ -75,11 +81,7 @@ struct FunctionProfile {
// determines the `UniqueBBID::CloneID` of the cloned blocks in that path.
SmallVector<SmallVector<unsigned>> ClonePaths;
// Cfg profile data (block and edge frequencies).
- CfgProfile Cfg;
- // Hash for each basic block. The Hashes are stored for every original block
- // (not cloned blocks), hence the map key being unsigned instead of
- // UniqueBBID.
- DenseMap<unsigned, uint64_t> BBHashes;
+ CFGProfile CFG;
};
class BasicBlockSectionsProfileReader {
@@ -103,19 +105,18 @@ class BasicBlockSectionsProfileReader {
SmallVector<SmallVector<unsigned>>
getClonePathsForFunction(StringRef FuncName) const;
- // Returns a pointer to the CfgProfile for the given function.
+ uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID,
+ const UniqueBBID &DestBBID) const;
+
+ // Returns a pointer to the CFGProfile for the function \p FuncName.
// Returns nullptr if no profile data is available for the function.
- const CfgProfile *getFunctionCfgProfile(StringRef FuncName) const {
- auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
- if (It == ProgramPathAndClusterInfo.end())
+ const CFGProfile *getFunctionCFGProfile(StringRef FuncName) const {
+ auto It = ProgramDirectivesAndProfile.find(getAliasName(FuncName));
+ if (It == ProgramDirectivesAndProfile.end())
return nullptr;
- return &It->second.Cfg;
+ return &It->second.CFG;
}
- // Return the complete function path and cluster info for the given function.
- std::pair<bool, FunctionPathAndClusterInfo>
- getFunctionPathAndClusterInfo(StringRef FuncName) const;
-
private:
StringRef getAliasName(StringRef FuncName) const {
auto R = FuncAliasMap.find(FuncName);
@@ -162,10 +163,10 @@ class BasicBlockSectionsProfileReader {
// for (all or some of) its basic blocks. The cluster information for every
// basic block includes its cluster ID along with the position of the basic
// block in that cluster.
- StringMap<FunctionProfile> ProgramPathAndClusterInfo;
+ StringMap<FunctionProfile> ProgramDirectivesAndProfile;
// Some functions have alias names. We use this map to find the main alias
- // name which appears in ProgramPathAndClusterInfo as a key.
+ // name which appears in ProgramDirectivesAndProfile as a key.
StringMap<StringRef> FuncAliasMap;
};
@@ -222,12 +223,11 @@ class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass {
SmallVector<SmallVector<unsigned>>
getClonePathsForFunction(StringRef FuncName) const;
+ const CFGProfile *getFunctionCFGProfile(StringRef FuncName) const;
+
uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID,
const UniqueBBID &DestBBID) const;
- std::pair<bool, FunctionPathAndClusterInfo>
- getFunctionPathAndClusterInfo(StringRef FuncName) const;
-
// Initializes the FunctionNameToDIFilename map for the current module and
// then reads the profile for the matching functions.
bool doInitialization(Module &M) override;
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index c76b8c3e74bec..14a96b3c64227 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1419,7 +1419,7 @@ static uint32_t getBBAddrMapMetadata(const MachineBasicBlock &MBB) {
static llvm::object::BBAddrMap::Features
getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
- bool HasCalls, const CfgProfile *FuncCfgProfile) {
+ bool HasCalls, const CFGProfile *FuncCFGProfile) {
// Ensure that the user has not passed in additional options while also
// specifying all or none.
if ((PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::None) ||
@@ -1441,7 +1441,7 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
bool BrProbEnabled =
AllFeatures ||
(!NoFeatures && PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::BrProb));
- bool PostLinkCfgEnabled = FuncCfgProfile && PgoAnalysisMapEmitBBSectionsCfg;
+ bool PostLinkCfgEnabled = FuncCFGProfile && PgoAnalysisMapEmitBBSectionsCfg;
if ((BBFreqEnabled || BrProbEnabled) && BBAddrMapSkipEmitBBEntries) {
MF.getFunction().getContext().emitError(
@@ -1464,9 +1464,9 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
if (auto *BBSPRPass =
getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>())
BBSPR = &BBSPRPass->getBBSPR();
- const CfgProfile *FuncCfgProfile = nullptr;
+ const CFGProfile *FuncCFGProfile = nullptr;
if (BBSPR)
- FuncCfgProfile = BBSPR->getFunctionCfgProfile(MF.getFunction().getName());
+ FuncCFGProfile = BBSPR->getFunctionCFGProfile(MF.getFunction().getName());
const MCSymbol *FunctionSymbol = getFunctionBegin();
@@ -1477,7 +1477,7 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
OutStreamer->emitInt8(BBAddrMapVersion);
OutStreamer->AddComment("feature");
auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size(), HasCalls,
- FuncCfgProfile);
+ FuncCFGProfile);
OutStreamer->emitInt8(Features.encode());
// Emit BB Information for each basic block in the function.
if (Features.MultiBBRange) {
@@ -1585,7 +1585,7 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
if (Features.PostLinkCfg) {
OutStreamer->AddComment("basic block frequency (propeller)");
OutStreamer->emitULEB128IntValue(
- FuncCfgProfile->getBlockCount(*MBB.getBBID()));
+ FuncCFGProfile->getBlockCount(*MBB.getBBID()));
}
}
if (Features.BrProb) {
@@ -1600,7 +1600,7 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
MBPI->getEdgeProbability(&MBB, SuccMBB).getNumerator());
if (Features.PostLinkCfg) {
OutStreamer->AddComment("successor branch frequency (propeller)");
- OutStreamer->emitULEB128IntValue(FuncCfgProfile->getEdgeCount(
+ OutStreamer->emitULEB128IntValue(FuncCFGProfile->getEdgeCount(
*MBB.getBBID(), *SuccMBB->getBBID()));
}
}
diff --git a/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp b/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
index 88c753ff66081..07b7b067e49b5 100644
--- a/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
+++ b/llvm/lib/CodeGen/BasicBlockMatchingAndInference.cpp
@@ -119,24 +119,23 @@ BasicBlockMatchingAndInference::initWeightInfoByMatching(MachineFunction &MF) {
StaleMatcher Matcher;
Matcher.init(Blocks, Hashes);
BasicBlockMatchingAndInference::WeightInfo MatchWeight;
- auto [IsValid, PathAndClusterInfo] =
- BSPR->getFunctionPathAndClusterInfo(MF.getName());
- if (!IsValid)
+ const CFGProfile *CFG = BSPR->getFunctionCFGProfile(MF.getName());
+ if (CFG == nullptr)
return MatchWeight;
- for (auto &BlockCount : PathAndClusterInfo.NodeCounts) {
- if (PathAndClusterInfo.BBHashes.count(BlockCount.first.BaseID)) {
- auto Hash = PathAndClusterInfo.BBHashes[BlockCount.first.BaseID];
+ for (auto &BlockCount : CFG->NodeCounts) {
+ if (CFG->BBHashes.count(BlockCount.first.BaseID)) {
+ auto Hash = CFG->BBHashes.lookup(BlockCount.first.BaseID);
MachineBasicBlock *Block = Matcher.matchBlock(BlendedBlockHash(Hash));
// When a basic block has clone copies, sum their counts.
if (Block != nullptr)
MatchWeight.BlockWeights[Block] += BlockCount.second;
}
}
- for (auto &PredItem : PathAndClusterInfo.EdgeCounts) {
+ for (auto &PredItem : CFG->EdgeCounts) {
auto PredID = PredItem.first.BaseID;
- if (!PathAndClusterInfo.BBHashes.count(PredID))
+ if (!CFG->BBHashes.count(PredID))
continue;
- auto PredHash = PathAndClusterInfo.BBHashes[PredID];
+ auto PredHash = CFG->BBHashes.lookup(PredID);
MachineBasicBlock *PredBlock =
Matcher.matchBlock(BlendedBlockHash(PredHash));
if (PredBlock == nullptr)
@@ -144,8 +143,8 @@ BasicBlockMatchingAndInference::initWeightInfoByMatching(MachineFunction &MF) {
for (auto &SuccItem : PredItem.second) {
auto SuccID = SuccItem.first.BaseID;
auto EdgeWeight = SuccItem.second;
- if (PathAndClusterInfo.BBHashes.count(SuccID)) {
- auto SuccHash = PathAndClusterInfo.BBHashes[SuccID];
+ if (CFG->BBHashes.count(SuccID)) {
+ auto SuccHash = CFG->BBHashes.lookup(SuccID);
MachineBasicBlock *SuccBlock =
Matcher.matchBlock(BlendedBlockHash(SuccHash));
// When an edge has clone copies, sum their counts.
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index 8baa478c7c5c0..9674ae4f76ff0 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -64,16 +64,16 @@ bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const {
SmallVector<BBClusterInfo>
BasicBlockSectionsProfileReader::getClusterInfoForFunction(
StringRef FuncName) const {
- auto R = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
- return R != ProgramPathAndClusterInfo.end() ? R->second.ClusterInfo
+ auto R = ProgramDirectivesAndProfile.find(getAliasName(FuncName));
+ return R != ProgramDirectivesAndProfile.end() ? R->second.ClusterInfo
: SmallVector<BBClusterInfo>();
}
SmallVector<SmallVector<unsigned>>
BasicBlockSectionsProfileReader::getClonePathsForFunction(
StringRef FuncName) const {
- auto R = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
- return R != ProgramPathAndClusterInfo.end()
+ auto R = ProgramDirectivesAndProfile.find(getAliasName(FuncName));
+ return R != ProgramDirectivesAndProfile.end()
? R->second.ClonePaths
: SmallVector<SmallVector<unsigned>>();
}
@@ -81,11 +81,10 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction(
uint64_t BasicBlockSectionsProfileReader::getEdgeCount(
StringRef FuncName, const UniqueBBID &SrcBBID,
const UniqueBBID &SinkBBID) const {
- auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
- if (It == ProgramPathAndClusterInfo.end())
- return 0;
- auto NodeIt = It->second.EdgeCounts.find(SrcBBID);
- if (NodeIt == It->second.EdgeCounts.end())
+ const CFGProfile *CFG = getFunctionCFGProfile(FuncName);
+ if (CFG == nullptr) return 0;
+ auto NodeIt = CFG->EdgeCounts.find(SrcBBID);
+ if (NodeIt == CFG->EdgeCounts.end())
return 0;
auto EdgeIt = NodeIt->second.find(SinkBBID);
if (EdgeIt == NodeIt->second.end())
@@ -93,15 +92,6 @@ uint64_t BasicBlockSectionsProfileReader::getEdgeCount(
return EdgeIt->second;
}
-std::pair<bool, FunctionPathAndClusterInfo>
-BasicBlockSectionsProfileReader::getFunctionPathAndClusterInfo(
- StringRef FuncName) const {
- auto R = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
- return R != ProgramPathAndClusterInfo.end()
- ? std::pair(true, R->second)
- : std::pair(false, FunctionPathAndClusterInfo());
-}
-
// Reads the version 1 basic block sections profile. Profile for each function
// is encoded as follows:
// m <module_name>
@@ -158,7 +148,7 @@ BasicBlockSectionsProfileReader::getFunctionPathAndClusterInfo(
// ....
// ****************************************************************************
Error BasicBlockSectionsProfileReader::ReadV1Profile() {
- auto FI = ProgramPathAndClusterInfo.end();
+ auto FI = ProgramDirectivesAndProfile.end();
// Current cluster ID corresponding to this function.
unsigned CurrentCluster = 0;
@@ -202,7 +192,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
if (!FunctionFound) {
// Skip the following profile by setting the profile iterator (FI) to
// the past-the-end element.
- FI = ProgramPathAndClusterInfo.end();
+ FI = ProgramDirectivesAndProfile.end();
DIFilename = "";
continue;
}
@@ -211,7 +201,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
// Prepare for parsing clusters of this function name.
// Start a new cluster map for this function name.
- auto R = ProgramPathAndClusterInfo.try_emplace(Values.front());
+ auto R = ProgramDirectivesAndProfile.try_emplace(Values.front());
// Report error when multiple profiles have been specified for the same
// function.
if (!R.second)
@@ -228,7 +218,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'c': // Basic block cluster specifier.
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramPathAndClusterInfo.end())
+ if (FI == ProgramDirectivesAndProfile.end())
continue;
// Reset current cluster position.
CurrentPosition = 0;
@@ -249,7 +239,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'p': { // Basic block cloning path specifier.
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramPathAndClusterInfo.end())
+ if (FI == ProgramDirectivesAndProfile.end())
continue;
SmallSet<unsigned, 5> BBsInPath;
FI->second.ClonePaths.push_back({});
@@ -269,7 +259,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'g': { // CFG profile specifier.
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramPathAndClusterInfo.end())
+ if (FI == ProgramDirectivesAndProfile.end())
continue;
// For each node, its CFG profile is encoded as
// <src>:<count>,<sink_1>:<count_1>,<sink_2>:<count_2>,...
@@ -290,10 +280,10 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
Twine("unsigned integer expected: '") + CountStr + "'");
if (i == 0) {
// The first element represents the source and its total count.
- FI->second.Cfg.NodeCounts[SrcBBID = *BBID] = Count;
+ FI->second.CFG.NodeCounts[SrcBBID = *BBID] = Count;
continue;
}
- FI->second.Cfg.EdgeCounts[SrcBBID][*BBID] = Count;
+ FI->second.CFG.EdgeCounts[SrcBBID][*BBID] = Count;
}
}
continue;
@@ -301,7 +291,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'h': { // Basic block hash secifier.
// Skip the profile when the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramPathAndClusterInfo.end())
+ if (FI == ProgramDirectivesAndProfile.end())
continue;
for (auto BBIDHashStr : Values) {
auto [BBIDStr, HashStr] = BBIDHashStr.split(':');
@@ -313,7 +303,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
return createProfileParseError(
Twine("unsigned integer expected in hex format: '") + HashStr +
"'");
- FI->second.BBHashes[BBID] = Hash;
+ FI->second.CFG.BBHashes[BBID] = Hash;
}
continue;
}
@@ -327,7 +317,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
}
Error BasicBlockSectionsProfileReader::ReadV0Profile() {
- auto FI = ProgramPathAndClusterInfo.end();
+ auto FI = ProgramDirectivesAndProfile.end();
// Current cluster ID corresponding to this function.
unsigned CurrentCluster = 0;
// Current position in the current cluster.
@@ -348,7 +338,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
if (S.consume_front("!")) {
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramPathAndClusterInfo.end())
+ if (FI == ProgramDirectivesAndProfile.end())
continue;
SmallVector<StringRef, 4> BBIDs;
S.split(BBIDs, ' ');
@@ -400,7 +390,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
if (!FunctionFound) {
// Skip the following profile by setting the profile iterator (FI) to
// the past-the-end element.
- FI = ProgramPathAndClusterInfo.end();
+ FI = ProgramDirectivesAndProfile.end();
continue;
}
for (size_t i = 1; i < Aliases.size(); ++i)
@@ -408,7 +398,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
// Prepare for parsing clusters of this function name.
// Start a new cluster map for this function name.
- auto R = ProgramPathAndClusterInfo.try_emplace(Aliases.front());
+ auto R = ProgramDirectivesAndProfile.try_emplace(Aliases.front());
// Report error when multiple profiles have been specified for the same
// function.
if (!R.second)
@@ -517,18 +507,16 @@ BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction(
return BBSPR.getClonePathsForFunction(FuncName);
}
+const CFGProfile *BasicBlockSectionsProfileReaderWrapperPass::getFunctionCFGProfile(StringRef FuncName) const {
+ return BBSPR.getFunctionCFGProfile(FuncName);
+}
+
uint64_t BasicBlockSectionsProfileReaderWrapperPass::getEdgeCount(
StringRef FuncName, const UniqueBBID &SrcBBID,
const UniqueBBID &SinkBBID) const {
return BBSPR.getEdgeCount(FuncName, SrcBBID, SinkBBID);
}
-std::pair<bool, FunctionPathAndClusterInfo>
-BasicBlockSectionsProfileReaderWrapperPass::getFunctionPathAndClusterInfo(
- StringRef FuncName) const {
- return BBSPR.getFunctionPathAndClusterInfo(FuncName);
-}
-
BasicBlockSectionsProfileReader &
BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() {
return BBSPR;
>From ff9e7169f17255453a6b3fc09c2583b85893352c Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 07:21:36 +0000
Subject: [PATCH 03/15] Update feature to 2 bytes.
---
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 2 +-
.../X86/basic-block-address-map-empty-function.ll | 6 +++---
.../X86/basic-block-address-map-function-sections.ll | 12 ++++++------
.../X86/basic-block-address-map-pgo-features.ll | 12 ++++++------
...ic-block-address-map-with-basic-block-sections.ll | 2 +-
.../X86/basic-block-address-map-with-emit-bb-hash.ll | 2 +-
.../CodeGen/X86/basic-block-address-map-with-mfs.ll | 4 ++--
llvm/test/CodeGen/X86/basic-block-address-map.ll | 2 +-
.../CodeGen/X86/basic-block-sections-pgo-features.ll | 4 ++--
9 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 14a96b3c64227..8ac7856d2fb56 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1478,7 +1478,7 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
OutStreamer->AddComment("feature");
auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size(), HasCalls,
FuncCFGProfile);
- OutStreamer->emitInt8(Features.encode());
+ OutStreamer->emitInt16(Features.encode());
// Emit BB Information for each basic block in the function.
if (Features.MultiBBRange) {
OutStreamer->AddComment("number of basic block ranges");
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll b/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
index 7457e3bdb24dc..c63022ed05c4e 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
@@ -19,7 +19,7 @@ entry:
; CHECK: func:
; CHECK: .Lfunc_begin1:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text{{$}}
-; CHECK-NEXT: .byte 5 # version
-; BASIC-NEXT: .byte 0 # feature
-; PGO-NEXT: .byte 3 # feature
+; CHECK-NEXT: .byte 5 # version
+; BASIC-NEXT: .short 0 # feature
+; PGO-NEXT: .short 3 # feature
; CHECK-NEXT: .quad .Lfunc_begin1 # function address
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll b/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
index bdfe1e4564fc1..892fa009935d2 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
@@ -10,8 +10,8 @@ define dso_local i32 @_Z3barv() {
; CHECK-LABEL: _Z3barv:
; CHECK-NEXT: [[BAR_BEGIN:.Lfunc_begin[0-9]+]]:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3barv{{$}}
-; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 0 # feature
+; CHECK-NEXT: .byte 5 # version
+; CHECK-NEXT: .short 0 # feature
; CHECK-NEXT: .quad [[BAR_BEGIN]] # function address
@@ -23,8 +23,8 @@ define dso_local i32 @_Z3foov() {
; CHECK-LABEL: _Z3foov:
; CHECK-NEXT: [[FOO_BEGIN:.Lfunc_begin[0-9]+]]:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3foov{{$}}
-; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 32 # feature
+; CHECK-NEXT: .byte 5 # version
+; CHECK-NEXT: .short 32 # feature
; CHECK-NEXT: .quad [[FOO_BEGIN]] # function address
@@ -36,6 +36,6 @@ define linkonce_odr dso_local i32 @_Z4fooTIiET_v() comdat {
; CHECK-LABEL: _Z4fooTIiET_v:
; CHECK-NEXT: [[FOOCOMDAT_BEGIN:.Lfunc_begin[0-9]+]]:
; CHECK: .section .llvm_bb_addr_map,"oG", at llvm_bb_addr_map,.text._Z4fooTIiET_v,_Z4fooTIiET_v,comdat{{$}}
-; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 0 # feature
+; CHECK-NEXT: .byte 5 # version
+; CHECK-NEXT: .short 0 # feature
; CHECK-NEXT: .quad [[FOOCOMDAT_BEGIN]] # function address
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
index ca3c8d18e7dc5..8dff2873b12c4 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
@@ -70,11 +70,11 @@ declare i32 @__gxx_personality_v0(...)
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text._Z3bazb{{$}}
; CHECK-NEXT: .byte 5 # version
-; BASIC-NEXT: .byte 32 # feature
-; PGO-ALL-NEXT: .byte 39 # feature
-; FEC-ONLY-NEXT:.byte 33 # feature
-; BBF-ONLY-NEXT:.byte 34 # feature
-; BRP-ONLY-NEXT:.byte 36 # feature
+; BASIC-NEXT: .short 32 # feature
+; PGO-ALL-NEXT: .short 39 # feature
+; FEC-ONLY-NEXT:.short 33 # feature
+; BBF-ONLY-NEXT:.short 34 # feature
+; BRP-ONLY-NEXT:.short 36 # feature
; CHECK-NEXT: .quad .Lfunc_begin0 # function address
; CHECK-NEXT: .byte 6 # number of basic blocks
; CHECK-NEXT: .byte 0 # BB id
@@ -146,7 +146,7 @@ declare i32 @__gxx_personality_v0(...)
; PGO-BRP-NEXT: .byte 5 # successor BB ID
; PGO-BRP-NEXT: .ascii "\200\200\200\200\b" # successor branch probability
-; SKIP-BB-ENTRIES: .byte 49 # feature
+; SKIP-BB-ENTRIES: .short 49 # feature
; SKIP-BB-ENTRIES-NEXT: .quad .Lfunc_begin0 # function address
; SKIP-BB-ENTRIES-NEXT: .byte 6 # number of basic blocks
; SKIP-BB-ENTRIES-NEXT: .byte 100 # function entry count
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
index 297cbd81ed036..16808819ab2d3 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll
@@ -48,7 +48,7 @@ declare i32 @__gxx_personality_v0(...)
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.hot._Z3bazb
; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 40 # feature
+; CHECK-NEXT: .short 40 # feature
; CHECK-NEXT: .byte 2 # number of basic block ranges
; CHECK-NEXT: .quad .Lfunc_begin0 # base address
; CHECK-NEXT: .byte 2 # number of basic blocks
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll
index f17e9a5849150..39d83cd26f316 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll
@@ -51,7 +51,7 @@ declare i32 @__gxx_personality_v0(...)
;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
; NOUNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text,unique,1
; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 96 # feature
+; CHECK-NEXT: .short 96 # feature
; CHECK-NEXT: .quad .Lfunc_begin0 # function address
; CHECK-NEXT: .byte 6 # number of basic blocks
; CHECK-NEXT: .byte 0 # BB id
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
index 36e21b9b3faa7..f8146bb38f71e 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll
@@ -59,8 +59,8 @@ declare i32 @qux()
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.hot.foo
; CHECK-NEXT: .byte 5 # version
-; BASIC-NEXT: .byte 40 # feature
-; PGO-NEXT: .byte 47 # feature
+; BASIC-NEXT: .short 40 # feature
+; PGO-NEXT: .short 47 # feature
; CHECK-NEXT: .byte 2 # number of basic block ranges
; CHECK-NEXT: .quad .Lfunc_begin0 # base address
; CHECK-NEXT: .byte 2 # number of basic blocks
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map.ll b/llvm/test/CodeGen/X86/basic-block-address-map.ll
index 523b939765525..5567ccd4f9e75 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map.ll
@@ -53,7 +53,7 @@ declare i32 @__gxx_personality_v0(...)
;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
; NOUNIQ: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text,unique,1
; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 32 # feature
+; CHECK-NEXT: .short 32 # feature
; CHECK-NEXT: .quad .Lfunc_begin0 # function address
; CHECK-NEXT: .byte 6 # number of basic blocks
; CHECK-NEXT: .byte 0 # BB id
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll
index 7f7d87369701b..5beb422919de5 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-pgo-features.ll
@@ -28,7 +28,7 @@ bb3:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.foo
; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 143 # feature
+; CHECK-NEXT: .short 143 # feature
; CHECK: .quad .Lfunc_begin0 # base address
; CHECK: .byte 0 # BB id
; CHECK: .byte 1 # BB id
@@ -80,7 +80,7 @@ bb2:
; CHECK: .section .llvm_bb_addr_map,"o", at llvm_bb_addr_map,.text.bar
; CHECK-NEXT: .byte 5 # version
-; CHECK-NEXT: .byte 7 # feature
+; CHECK-NEXT: .short 7 # feature
; CHECK: .quad .Lfunc_begin1 # function address
; CHECK: .byte 0 # BB id
; CHECK: .byte 1 # BB id
>From 4d06f1ba7b83dedcb656638940a3f710bae52c97 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 07:21:59 +0000
Subject: [PATCH 04/15] clang-format.
---
.../llvm/CodeGen/BasicBlockSectionsProfileReader.h | 1 -
llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp | 9 ++++++---
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 898b36ae035d1..c33b13602e027 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -54,7 +54,6 @@ struct CFGProfile {
// UniqueBBID.
DenseMap<unsigned, uint64_t> BBHashes;
-
// Returns the profile count for the given basic block or zero if it does not
// exist.
uint64_t getBlockCount(const UniqueBBID &BBID) const {
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index 9674ae4f76ff0..76ed2ea541ea5 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -66,7 +66,7 @@ BasicBlockSectionsProfileReader::getClusterInfoForFunction(
StringRef FuncName) const {
auto R = ProgramDirectivesAndProfile.find(getAliasName(FuncName));
return R != ProgramDirectivesAndProfile.end() ? R->second.ClusterInfo
- : SmallVector<BBClusterInfo>();
+ : SmallVector<BBClusterInfo>();
}
SmallVector<SmallVector<unsigned>>
@@ -82,7 +82,8 @@ uint64_t BasicBlockSectionsProfileReader::getEdgeCount(
StringRef FuncName, const UniqueBBID &SrcBBID,
const UniqueBBID &SinkBBID) const {
const CFGProfile *CFG = getFunctionCFGProfile(FuncName);
- if (CFG == nullptr) return 0;
+ if (CFG == nullptr)
+ return 0;
auto NodeIt = CFG->EdgeCounts.find(SrcBBID);
if (NodeIt == CFG->EdgeCounts.end())
return 0;
@@ -507,7 +508,9 @@ BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction(
return BBSPR.getClonePathsForFunction(FuncName);
}
-const CFGProfile *BasicBlockSectionsProfileReaderWrapperPass::getFunctionCFGProfile(StringRef FuncName) const {
+const CFGProfile *
+BasicBlockSectionsProfileReaderWrapperPass::getFunctionCFGProfile(
+ StringRef FuncName) const {
return BBSPR.getFunctionCFGProfile(FuncName);
}
>From 13d94b22861a99b23eded233594cd75c675ec253 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 07:46:40 +0000
Subject: [PATCH 05/15] Fix comments after renaming
---
llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index c33b13602e027..1c96b484a2090 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -158,7 +158,7 @@ class BasicBlockSectionsProfileReader {
// This contains the BB cluster information for the whole program.
//
- // For every function name, it contains the cloning and cluster information
+ // For every function name, it contains the cloning directives and profile data
// for (all or some of) its basic blocks. The cluster information for every
// basic block includes its cluster ID along with the position of the basic
// block in that cluster.
>From 97c4f1ed6b57a84d4c9c890a6493df691d0a9c44 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 07:59:36 +0000
Subject: [PATCH 06/15] clang-format.
---
.../llvm/CodeGen/BasicBlockSectionsProfileReader.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 1c96b484a2090..59401ed27f901 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -158,10 +158,10 @@ class BasicBlockSectionsProfileReader {
// This contains the BB cluster information for the whole program.
//
- // For every function name, it contains the cloning directives and profile data
- // for (all or some of) its basic blocks. The cluster information for every
- // basic block includes its cluster ID along with the position of the basic
- // block in that cluster.
+ // For every function name, it contains the cloning directives and profile
+ // data for (all or some of) its basic blocks. The cluster information for
+ // every basic block includes its cluster ID along with the position of the
+ // basic block in that cluster.
StringMap<FunctionProfile> ProgramDirectivesAndProfile;
// Some functions have alias names. We use this map to find the main alias
>From 550013e220f2b4cfc9d90e78a900998465692faf Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 17:18:07 +0000
Subject: [PATCH 07/15] Rename FunctionProfile and ProgramDirectivesAndProfile
---
.../CodeGen/BasicBlockSectionsProfileReader.h | 8 ++---
.../BasicBlockSectionsProfileReader.cpp | 30 +++++++++----------
2 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 59401ed27f901..4c3c4b0cff126 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -71,8 +71,8 @@ struct CFGProfile {
}
};
-// This represents the raw input profile for one function.
-struct FunctionProfile {
+// This represents the raw optimization profile for one function.
+struct FunctionOptimizationProfile {
// BB Cluster information specified by `UniqueBBID`s.
SmallVector<BBClusterInfo> ClusterInfo;
// Paths to clone. A path a -> b -> c -> d implies cloning b, c, and d along
@@ -162,10 +162,10 @@ class BasicBlockSectionsProfileReader {
// data for (all or some of) its basic blocks. The cluster information for
// every basic block includes its cluster ID along with the position of the
// basic block in that cluster.
- StringMap<FunctionProfile> ProgramDirectivesAndProfile;
+ StringMap<FunctionOptimizationProfile> ProgramOptimizationProfile;
// Some functions have alias names. We use this map to find the main alias
- // name which appears in ProgramDirectivesAndProfile as a key.
+ // name which appears in ProgramOptimizationProfile as a key.
StringMap<StringRef> FuncAliasMap;
};
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index 76ed2ea541ea5..cfffb20fa96e2 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -64,16 +64,16 @@ bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const {
SmallVector<BBClusterInfo>
BasicBlockSectionsProfileReader::getClusterInfoForFunction(
StringRef FuncName) const {
- auto R = ProgramDirectivesAndProfile.find(getAliasName(FuncName));
- return R != ProgramDirectivesAndProfile.end() ? R->second.ClusterInfo
+ auto R = ProgramOptimizationProfile.find(getAliasName(FuncName));
+ return R != ProgramOptimizationProfile.end() ? R->second.ClusterInfo
: SmallVector<BBClusterInfo>();
}
SmallVector<SmallVector<unsigned>>
BasicBlockSectionsProfileReader::getClonePathsForFunction(
StringRef FuncName) const {
- auto R = ProgramDirectivesAndProfile.find(getAliasName(FuncName));
- return R != ProgramDirectivesAndProfile.end()
+ auto R = ProgramOptimizationProfile.find(getAliasName(FuncName));
+ return R != ProgramOptimizationProfile.end()
? R->second.ClonePaths
: SmallVector<SmallVector<unsigned>>();
}
@@ -149,7 +149,7 @@ uint64_t BasicBlockSectionsProfileReader::getEdgeCount(
// ....
// ****************************************************************************
Error BasicBlockSectionsProfileReader::ReadV1Profile() {
- auto FI = ProgramDirectivesAndProfile.end();
+ auto FI = ProgramOptimizationProfile.end();
// Current cluster ID corresponding to this function.
unsigned CurrentCluster = 0;
@@ -193,7 +193,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
if (!FunctionFound) {
// Skip the following profile by setting the profile iterator (FI) to
// the past-the-end element.
- FI = ProgramDirectivesAndProfile.end();
+ FI = ProgramOptimizationProfile.end();
DIFilename = "";
continue;
}
@@ -202,7 +202,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
// Prepare for parsing clusters of this function name.
// Start a new cluster map for this function name.
- auto R = ProgramDirectivesAndProfile.try_emplace(Values.front());
+ auto R = ProgramOptimizationProfile.try_emplace(Values.front());
// Report error when multiple profiles have been specified for the same
// function.
if (!R.second)
@@ -219,7 +219,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'c': // Basic block cluster specifier.
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramDirectivesAndProfile.end())
+ if (FI == ProgramOptimizationProfile.end())
continue;
// Reset current cluster position.
CurrentPosition = 0;
@@ -240,7 +240,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'p': { // Basic block cloning path specifier.
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramDirectivesAndProfile.end())
+ if (FI == ProgramOptimizationProfile.end())
continue;
SmallSet<unsigned, 5> BBsInPath;
FI->second.ClonePaths.push_back({});
@@ -260,7 +260,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'g': { // CFG profile specifier.
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramDirectivesAndProfile.end())
+ if (FI == ProgramOptimizationProfile.end())
continue;
// For each node, its CFG profile is encoded as
// <src>:<count>,<sink_1>:<count_1>,<sink_2>:<count_2>,...
@@ -292,7 +292,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
case 'h': { // Basic block hash secifier.
// Skip the profile when the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramDirectivesAndProfile.end())
+ if (FI == ProgramOptimizationProfile.end())
continue;
for (auto BBIDHashStr : Values) {
auto [BBIDStr, HashStr] = BBIDHashStr.split(':');
@@ -318,7 +318,7 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
}
Error BasicBlockSectionsProfileReader::ReadV0Profile() {
- auto FI = ProgramDirectivesAndProfile.end();
+ auto FI = ProgramOptimizationProfile.end();
// Current cluster ID corresponding to this function.
unsigned CurrentCluster = 0;
// Current position in the current cluster.
@@ -339,7 +339,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
if (S.consume_front("!")) {
// Skip the profile when we the profile iterator (FI) refers to the
// past-the-end element.
- if (FI == ProgramDirectivesAndProfile.end())
+ if (FI == ProgramOptimizationProfile.end())
continue;
SmallVector<StringRef, 4> BBIDs;
S.split(BBIDs, ' ');
@@ -391,7 +391,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
if (!FunctionFound) {
// Skip the following profile by setting the profile iterator (FI) to
// the past-the-end element.
- FI = ProgramDirectivesAndProfile.end();
+ FI = ProgramOptimizationProfile.end();
continue;
}
for (size_t i = 1; i < Aliases.size(); ++i)
@@ -399,7 +399,7 @@ Error BasicBlockSectionsProfileReader::ReadV0Profile() {
// Prepare for parsing clusters of this function name.
// Start a new cluster map for this function name.
- auto R = ProgramDirectivesAndProfile.try_emplace(Aliases.front());
+ auto R = ProgramOptimizationProfile.try_emplace(Aliases.front());
// Report error when multiple profiles have been specified for the same
// function.
if (!R.second)
>From a76999799cf7b357d176183f74255730758c0d96 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 17:20:21 +0000
Subject: [PATCH 08/15] Fix remaining references to ProgramDirectivesAndProfile
in BasicBlockSectionsProfileReader.h
---
llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 4c3c4b0cff126..f3743f00c6237 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -110,8 +110,8 @@ class BasicBlockSectionsProfileReader {
// Returns a pointer to the CFGProfile for the function \p FuncName.
// Returns nullptr if no profile data is available for the function.
const CFGProfile *getFunctionCFGProfile(StringRef FuncName) const {
- auto It = ProgramDirectivesAndProfile.find(getAliasName(FuncName));
- if (It == ProgramDirectivesAndProfile.end())
+ auto It = ProgramOptimizationProfile.find(getAliasName(FuncName));
+ if (It == ProgramOptimizationProfile.end())
return nullptr;
return &It->second.CFG;
}
>From 02db9dbbb9db81cb08d36e8c94547d6e965e81c9 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 17:24:42 +0000
Subject: [PATCH 09/15] Update comment for ProgramOptimizationProfile
---
.../llvm/CodeGen/BasicBlockSectionsProfileReader.h | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index f3743f00c6237..9634b49c02a1b 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -156,12 +156,10 @@ class BasicBlockSectionsProfileReader {
// empty string if no debug info is available.
StringMap<SmallString<128>> FunctionNameToDIFilename;
- // This contains the BB cluster information for the whole program.
- //
- // For every function name, it contains the cloning directives and profile
- // data for (all or some of) its basic blocks. The cluster information for
- // every basic block includes its cluster ID along with the position of the
- // basic block in that cluster.
+ // This map contains the optimization profile for each function in the program.
+ // A function's optimization profile consists of CFG data (node and edge
+ // counts) and layout directives such as basic block clustering and cloning
+ // paths.
StringMap<FunctionOptimizationProfile> ProgramOptimizationProfile;
// Some functions have alias names. We use this map to find the main alias
>From 3d92df1b639121822f82d53cfd3812ed40a5490d Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 17:29:14 +0000
Subject: [PATCH 10/15] Shorten comment for FunctionOptimizationProfile struct
---
llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index 9634b49c02a1b..d74c3c7e60dca 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -71,7 +71,8 @@ struct CFGProfile {
}
};
-// This represents the raw optimization profile for one function.
+// This struct represents the raw optimization profile for a function,
+// including CFG data (block and edge counts) and layout directives (clustering and cloning paths).
struct FunctionOptimizationProfile {
// BB Cluster information specified by `UniqueBBID`s.
SmallVector<BBClusterInfo> ClusterInfo;
>From 828f49673b0e592749a3695b7789207c686588c8 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 18:09:57 +0000
Subject: [PATCH 11/15] Docs: Add post-link CFG map feature to Extensions.rst
---
llvm/docs/Extensions.rst | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst
index 0d7f599548fb7..065442d2c2381 100644
--- a/llvm/docs/Extensions.rst
+++ b/llvm/docs/Extensions.rst
@@ -526,6 +526,13 @@ those bits are:
defined in ``llvm/Support/BranchProbability.h``. It indicates the probability
that the block is followed by a given successor block during execution.
+#. Post-Link CFG - When enabled, the PGO Analysis Map will include CFG
+ information obtained from a post-link tool, such as Propeller. This feature
+ is enabled with the ``-pgo-analysis-map-emit-bb-sections-cfg`` flag. When
+ this option is active, the map will contain basic block and edge frequencies
+ from the basic block sections profile. This provides more accurate profiling
+ information that reflects the final binary layout.
+
This extra data requires version 2 or above. This is necessary since successors
of basic blocks won't know their index but will know their BB ID.
>From f5ec2de8caae0b1fcfdb1f76dbd5ca96696a800b Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 18:16:39 +0000
Subject: [PATCH 12/15] Docs: Add note about BBAddrMap version 5 feature data
length
---
llvm/docs/Extensions.rst | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst
index 065442d2c2381..7611b9c72826f 100644
--- a/llvm/docs/Extensions.rst
+++ b/llvm/docs/Extensions.rst
@@ -419,6 +419,9 @@ features of the older versions).
Version 4 (newest): Capable of encoding basic block hashes. This feature is
enabled by the 7th bit of the feature byte.
+Starting from version 5, the feature data will be two bytes long.
+
+
Example:
.. code-block:: gas
>From e25f9442fb22acfedb31ee5c851ccdc8304abf89 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 18:21:30 +0000
Subject: [PATCH 13/15] clang-format.
---
.../llvm/CodeGen/BasicBlockSectionsProfileReader.h | 11 ++++++-----
llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp | 2 +-
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
index d74c3c7e60dca..b3e33b911381a 100644
--- a/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
+++ b/llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h
@@ -72,7 +72,8 @@ struct CFGProfile {
};
// This struct represents the raw optimization profile for a function,
-// including CFG data (block and edge counts) and layout directives (clustering and cloning paths).
+// including CFG data (block and edge counts) and layout directives (clustering
+// and cloning paths).
struct FunctionOptimizationProfile {
// BB Cluster information specified by `UniqueBBID`s.
SmallVector<BBClusterInfo> ClusterInfo;
@@ -157,10 +158,10 @@ class BasicBlockSectionsProfileReader {
// empty string if no debug info is available.
StringMap<SmallString<128>> FunctionNameToDIFilename;
- // This map contains the optimization profile for each function in the program.
- // A function's optimization profile consists of CFG data (node and edge
- // counts) and layout directives such as basic block clustering and cloning
- // paths.
+ // This map contains the optimization profile for each function in the
+ // program. A function's optimization profile consists of CFG data (node and
+ // edge counts) and layout directives such as basic block clustering and
+ // cloning paths.
StringMap<FunctionOptimizationProfile> ProgramOptimizationProfile;
// Some functions have alias names. We use this map to find the main alias
diff --git a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
index cfffb20fa96e2..68f380b286644 100644
--- a/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp
@@ -66,7 +66,7 @@ BasicBlockSectionsProfileReader::getClusterInfoForFunction(
StringRef FuncName) const {
auto R = ProgramOptimizationProfile.find(getAliasName(FuncName));
return R != ProgramOptimizationProfile.end() ? R->second.ClusterInfo
- : SmallVector<BBClusterInfo>();
+ : SmallVector<BBClusterInfo>();
}
SmallVector<SmallVector<unsigned>>
>From f2df8934d732ddad443ad33b153cef8acab1584a Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 18:43:18 +0000
Subject: [PATCH 14/15] Docs: Reorder BBAddrMap versions and elaborate on
version 5 features
---
llvm/docs/Extensions.rst | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst
index 7611b9c72826f..ac76407b02812 100644
--- a/llvm/docs/Extensions.rst
+++ b/llvm/docs/Extensions.rst
@@ -416,10 +416,13 @@ as offsets relative to prior addresses.
The following versioning schemes are currently supported (newer versions support
features of the older versions).
-Version 4 (newest): Capable of encoding basic block hashes. This feature is
+Version 4: Capable of encoding basic block hashes. This feature is
enabled by the 7th bit of the feature byte.
-Starting from version 5, the feature data will be two bytes long.
+Version 5 (newest): The feature data will be two bytes long to accommodate
+future extensions. It is also capable of encoding Post-Link CFG information,
+which provides basic block and edge frequencies obtained from a post-link tool
+like Propeller, reflecting the final binary layout.
Example:
>From e93a0f991da8e466a5453269037b3c047bad28f4 Mon Sep 17 00:00:00 2001
From: Rahman Lavaee <rahmanl at google.com>
Date: Thu, 11 Dec 2025 18:46:18 +0000
Subject: [PATCH 15/15] Refine extensions.rst
---
llvm/docs/Extensions.rst | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst
index ac76407b02812..e0d05a3872539 100644
--- a/llvm/docs/Extensions.rst
+++ b/llvm/docs/Extensions.rst
@@ -416,15 +416,15 @@ as offsets relative to prior addresses.
The following versioning schemes are currently supported (newer versions support
features of the older versions).
+Version 5 (newest): Capable of encoding Post-Link CFG information, which
+provides basic block and edge frequencies obtained from a post-link tool like
+Propeller, reflecting the final binary layout. This feature is enabled by the 8th
+bit of the feature entry.
+The feature data will be two bytes long to accommodate future extensions.
+
Version 4: Capable of encoding basic block hashes. This feature is
enabled by the 7th bit of the feature byte.
-Version 5 (newest): The feature data will be two bytes long to accommodate
-future extensions. It is also capable of encoding Post-Link CFG information,
-which provides basic block and edge frequencies obtained from a post-link tool
-like Propeller, reflecting the final binary layout.
-
-
Example:
.. code-block:: gas
More information about the llvm-commits
mailing list