[llvm] [PseudoProbe] Encode inlining attributes to probe descriptor (PR #177695)
Wei Wang via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 22:06:26 PDT 2026
https://github.com/apolloww updated https://github.com/llvm/llvm-project/pull/177695
>From cd83a1150516123ae5820678054ceb8e5692e3b1 Mon Sep 17 00:00:00 2001
From: Wei Wang <apollo.mobility at gmail.com>
Date: Thu, 22 Jan 2026 15:26:35 -0800
Subject: [PATCH 1/2] Encode always_inline to probe desc
---
llvm/include/llvm/IR/MDBuilder.h | 2 +-
llvm/include/llvm/IR/PseudoProbe.h | 24 +++-
llvm/include/llvm/MC/MCPseudoProbe.h | 7 +-
.../Utils/SampleProfileLoaderBaseImpl.h | 5 +-
llvm/lib/IR/MDBuilder.cpp | 8 +-
llvm/lib/MC/MCPseudoProbe.cpp | 13 +-
llvm/lib/Target/TargetLoweringObjectFile.cpp | 8 +-
.../lib/Transforms/IPO/SampleProfileProbe.cpp | 10 +-
llvm/test/CodeGen/X86/fsafdo_probe.ll | 2 +-
llvm/test/CodeGen/X86/fsafdo_probe2.ll | 8 +-
.../CodeGen/X86/pseudo-probe-desc-check.ll | 2 +-
.../ThinLTO/X86/pseudo-probe-desc-import.ll | 4 +-
.../csspgo-import-list-callee-samples.ll | 6 +-
.../csspgo-import-list-preinliner.ll | 2 +-
.../csspgo-profile-checksum-mismatch-attr.ll | 6 +-
.../profile-correlation-irreducible-loops.ll | 4 +-
...ofile-inference-even-count-distribution.ll | 8 +-
.../profile-inference-islands.ll | 6 +-
.../profile-inference-rebalance-large.ll | 8 +-
.../profile-inference-rebalance.ll | 8 +-
.../SampleProfile/profile-inference.ll | 8 +-
.../pseudo-probe-callee-profile-mismatch.ll | 2 +-
.../SampleProfile/pseudo-probe-desc-attr.ll | 48 +++++++
.../SampleProfile/pseudo-probe-desc-guid.ll | 4 +-
.../SampleProfile/pseudo-probe-emit.ll | 123 ++++++++++++++----
.../SampleProfile/pseudo-probe-icp-factor.ll | 8 +-
.../SampleProfile/pseudo-probe-inline.ll | 4 +-
.../SampleProfile/pseudo-probe-instsched.ll | 3 +-
.../SampleProfile/pseudo-probe-invoke.ll | 2 +-
.../pseudo-probe-missing-probe.ll | 6 +-
.../pseudo-probe-profile-metadata-2.ll | 2 +-
.../pseudo-probe-profile-metadata.ll | 2 +-
.../pseudo-probe-profile-mismatch.ll | 8 +-
...pseudo-probe-stale-profile-matching-LCS.ll | 4 +-
...pseudo-probe-stale-profile-matching-lto.ll | 4 +-
.../pseudo-probe-stale-profile-matching.ll | 6 +-
...udo-probe-stale-profile-name-similarity.ll | 4 +-
...-probe-stale-profile-renaming-recursive.ll | 6 +-
.../pseudo-probe-stale-profile-renaming.ll | 14 +-
.../pseudo-probe-stale-profile-toplev-func.ll | 6 +-
40 files changed, 283 insertions(+), 122 deletions(-)
create mode 100644 llvm/test/Transforms/SampleProfile/pseudo-probe-desc-attr.ll
diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h
index e88de73567c35..a4e09bbd14715 100644
--- a/llvm/include/llvm/IR/MDBuilder.h
+++ b/llvm/include/llvm/IR/MDBuilder.h
@@ -111,7 +111,7 @@ class MDBuilder {
/// Return metadata containing the pseudo probe descriptor for a function.
LLVM_ABI MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash,
- StringRef FName);
+ uint8_t Attributes, StringRef FName);
/// Return metadata containing llvm statistics.
LLVM_ABI MDNode *
diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h
index 5c652b14650d5..9b2aa1ad4c895 100644
--- a/llvm/include/llvm/IR/PseudoProbe.h
+++ b/llvm/include/llvm/IR/PseudoProbe.h
@@ -35,6 +35,12 @@ enum class PseudoProbeAttributes {
HasDiscriminator = 0x4, // for probes with a discriminator
};
+enum class PseudoProbeDescAttributes {
+ AlwaysInline = 0x1, // Function has always_inline attribute
+ NoInline = 0x2, // Function has no_inline attribute
+ Sentinel = 0x10, // Guard against the 4bit encoding space
+};
+
// The saturated distrution factor representing 100% for block probes.
constexpr static uint64_t PseudoProbeFullDistributionFactor =
std::numeric_limits<uint64_t>::max();
@@ -107,12 +113,20 @@ struct PseudoProbeDwarfDiscriminator {
class PseudoProbeDescriptor {
uint64_t FunctionGUID;
uint64_t FunctionHash;
+ uint8_t Attributes;
public:
- PseudoProbeDescriptor(uint64_t GUID, uint64_t Hash)
- : FunctionGUID(GUID), FunctionHash(Hash) {}
+ PseudoProbeDescriptor(uint64_t GUID, uint64_t Hash, uint8_t Attributes)
+ : FunctionGUID(GUID), FunctionHash(Hash), Attributes(Attributes) {
+ assert((Attributes <
+ static_cast<uint8_t>(PseudoProbeDescAttributes::Sentinel)) &&
+ "Attributes exceed 4bit long encoding space.");
+ }
uint64_t getFunctionGUID() const { return FunctionGUID; }
uint64_t getFunctionHash() const { return FunctionHash; }
+ bool hasAttribute(PseudoProbeDescAttributes Attribute) const {
+ return Attributes & static_cast<uint8_t>(Attribute);
+ }
};
struct PseudoProbe {
@@ -120,9 +134,9 @@ struct PseudoProbe {
uint32_t Type;
uint32_t Attr;
uint32_t Discriminator;
- // Distribution factor that estimates the portion of the real execution count.
- // A saturated distribution factor stands for 1.0 or 100%. A pesudo probe has
- // a factor with the value ranged from 0.0 to 1.0.
+ // Distribution factor that estimates the portion of the real execution
+ // count. A saturated distribution factor stands for 1.0 or 100%. A pesudo
+ // probe has a factor with the value ranged from 0.0 to 1.0.
float Factor;
};
diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h
index 3799d2d84631d..61dc47ecf268c 100644
--- a/llvm/include/llvm/MC/MCPseudoProbe.h
+++ b/llvm/include/llvm/MC/MCPseudoProbe.h
@@ -88,10 +88,13 @@ enum class MCPseudoProbeFlag {
struct MCPseudoProbeFuncDesc {
uint64_t FuncGUID = 0;
uint64_t FuncHash = 0;
+ uint8_t Attributes = 0;
StringRef FuncName;
- MCPseudoProbeFuncDesc(uint64_t GUID, uint64_t Hash, StringRef Name)
- : FuncGUID(GUID), FuncHash(Hash), FuncName(Name){};
+ MCPseudoProbeFuncDesc(uint64_t GUID, uint64_t Hash, uint8_t Attributes,
+ StringRef Name)
+ : FuncGUID(GUID), FuncHash(Hash), Attributes(Attributes),
+ FuncName(Name) {};
LLVM_ABI void print(raw_ostream &OS);
};
diff --git a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
index 236b78f2c27a1..36dca1a858053 100644
--- a/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
+++ b/llvm/include/llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h
@@ -99,7 +99,10 @@ class PseudoProbeManager {
->getZExtValue();
auto Hash = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1))
->getZExtValue();
- GUIDToProbeDescMap.try_emplace(GUID, PseudoProbeDescriptor(GUID, Hash));
+ auto Attributes = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2))
+ ->getZExtValue();
+ GUIDToProbeDescMap.try_emplace(
+ GUID, PseudoProbeDescriptor(GUID, Hash, Attributes));
}
for (const auto &Func : M) {
if (Func.hasWeakLinkage() || Func.hasExternalWeakLinkage()) {
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index 9a8e417b43ba7..a573af6c0e7fe 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -349,12 +349,14 @@ MDNode *MDBuilder::createIrrLoopHeaderWeight(uint64_t Weight) {
}
MDNode *MDBuilder::createPseudoProbeDesc(uint64_t GUID, uint64_t Hash,
- StringRef FName) {
+ uint8_t Attributes, StringRef FName) {
auto *Int64Ty = Type::getInt64Ty(Context);
- SmallVector<Metadata *, 3> Ops(3);
+ auto *Int8Ty = Type::getInt8Ty(Context);
+ SmallVector<Metadata *, 4> Ops(4);
Ops[0] = createConstant(ConstantInt::get(Int64Ty, GUID));
Ops[1] = createConstant(ConstantInt::get(Int64Ty, Hash));
- Ops[2] = createString(FName);
+ Ops[2] = createConstant(ConstantInt::get(Int8Ty, Attributes));
+ Ops[3] = createString(FName);
return MDNode::get(Context, Ops);
}
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
index 11e42118a29ef..68f1b61b1f512 100644
--- a/llvm/lib/MC/MCPseudoProbe.cpp
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -277,7 +277,8 @@ static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP,
void MCPseudoProbeFuncDesc::print(raw_ostream &OS) {
OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n";
- OS << "Hash: " << FuncHash << "\n";
+ OS << "Hash: " << FuncHash
+ << " Attributes: " << static_cast<unsigned>(Attributes) << "\n";
}
void MCDecodedPseudoProbe::getInlineContext(
@@ -378,7 +379,7 @@ bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,
// The pseudo_probe_desc section has a format like:
// .section .pseudo_probe_desc,"", at progbits
// .quad -5182264717993193164 // GUID
- // .quad 4294967295 // Hash
+ // .quad 4294967295 // Hash and attributes
// .uleb 3 // Name size
// .ascii "foo" // Name
// .quad -2624081020897602054
@@ -394,7 +395,7 @@ bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,
// GUID
if (!readUnencodedNumber<uint64_t>())
return false;
- // Hash
+ // Hash and attributes
if (!readUnencodedNumber<uint64_t>())
return false;
@@ -421,8 +422,10 @@ bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,
StringRef Name = cantFail(errorOrToExpected(readString(NameSize)));
// Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap
- GUID2FuncDescMap.emplace_back(
- GUID, Hash, IsMMapped ? Name : Name.copy(FuncNameAllocator));
+ // [60:63] bits in Hash are encoded with attributes.
+ GUID2FuncDescMap.emplace_back(GUID, Hash & ((1ULL << 60) - 1), Hash >> 60,
+ IsMMapped ? Name
+ : Name.copy(FuncNameAllocator));
}
assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");
assert(GUID2FuncDescMap.size() == FuncDescCount &&
diff --git a/llvm/lib/Target/TargetLoweringObjectFile.cpp b/llvm/lib/Target/TargetLoweringObjectFile.cpp
index 83acd956b7a7b..ab82f63885c27 100644
--- a/llvm/lib/Target/TargetLoweringObjectFile.cpp
+++ b/llvm/lib/Target/TargetLoweringObjectFile.cpp
@@ -210,7 +210,8 @@ void TargetLoweringObjectFile::emitPseudoProbeDescMetadata(
const auto *MD = cast<MDNode>(Operand);
auto *GUID = mdconst::extract<ConstantInt>(MD->getOperand(0));
auto *Hash = mdconst::extract<ConstantInt>(MD->getOperand(1));
- auto *Name = cast<MDString>(MD->getOperand(2));
+ auto *Attributes = mdconst::extract<ConstantInt>(MD->getOperand(2));
+ auto *Name = cast<MDString>(MD->getOperand(3));
auto *S = C.getObjectFileInfo()->getPseudoProbeDescSection(
TM->getFunctionSections() ? Name->getString() : StringRef());
@@ -221,7 +222,10 @@ void TargetLoweringObjectFile::emitPseudoProbeDescMetadata(
COMDATSymEmitter(Streamer);
Streamer.emitInt64(GUID->getZExtValue());
- Streamer.emitInt64(Hash->getZExtValue());
+ // [0:59] - function hash
+ // [60:63] - attributes
+ Streamer.emitInt64(Hash->getZExtValue() |
+ (Attributes->getZExtValue() << 60));
Streamer.emitULEB128IntValue(Name->getString().size());
Streamer.emitBytes(Name->getString());
}
diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
index 7fd7d4d4f750b..8f3a2e026bd48 100644
--- a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
@@ -438,10 +438,16 @@ void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) {
// Create module-level metadata that contains function info necessary to
// synthesize probe-based sample counts, which are
// - FunctionGUID
- // - FunctionHash.
+ // - FunctionHash
+ // - Attributes
// - FunctionName
auto Hash = getFunctionHash();
- auto *MD = MDB.createPseudoProbeDesc(Guid, Hash, FName);
+ uint8_t Attributes = 0;
+ if (F.hasFnAttribute(Attribute::AlwaysInline))
+ Attributes |= static_cast<uint8_t>(PseudoProbeDescAttributes::AlwaysInline);
+ if (F.hasFnAttribute(Attribute::NoInline))
+ Attributes |= static_cast<uint8_t>(PseudoProbeDescAttributes::NoInline);
+ auto *MD = MDB.createPseudoProbeDesc(Guid, Hash, Attributes, FName);
auto *NMD = M->getNamedMetadata(PseudoProbeDescMetadataName);
assert(NMD && "llvm.pseudo_probe_desc should be pre-created");
NMD->addOperand(MD);
diff --git a/llvm/test/CodeGen/X86/fsafdo_probe.ll b/llvm/test/CodeGen/X86/fsafdo_probe.ll
index fa91670306ba1..d7f0e6bc30b30 100644
--- a/llvm/test/CodeGen/X86/fsafdo_probe.ll
+++ b/llvm/test/CodeGen/X86/fsafdo_probe.ll
@@ -70,4 +70,4 @@ attributes #0 = { inaccessiblememonly nocallback nofree nosync nounwind willretu
!10 = !DILocation(line: 7, column: 3, scope: !9)
!11 = !DILocation(line: 9, column: 5, scope: !9)
!12 = !DILocation(line: 14, column: 3, scope: !6)
-!13 = !{i64 6699318081062747564, i64 138464321060, !"foo"}
+!13 = !{i64 6699318081062747564, i64 138464321060, i8 0, !"foo"}
diff --git a/llvm/test/CodeGen/X86/fsafdo_probe2.ll b/llvm/test/CodeGen/X86/fsafdo_probe2.ll
index 04e4145abdc0c..f2895bfb696f3 100644
--- a/llvm/test/CodeGen/X86/fsafdo_probe2.ll
+++ b/llvm/test/CodeGen/X86/fsafdo_probe2.ll
@@ -263,10 +263,10 @@ attributes #7 = { nocallback nofree nosync nounwind speculatable willreturn memo
!8 = !{i32 1, !"wchar_size", i32 4}
!9 = !{i32 7, !"uwtable", i32 2}
!10 = !{!"clang version 17.0.0 (https://github.com/llvm/llvm-project.git fb16df500443aa5129f4a5e4dc4d9dcac613a809)"}
-!11 = !{i64 -2012135647395072713, i64 4294967295, !"bar"}
-!12 = !{i64 9204417991963109735, i64 72617220756, !"work"}
-!13 = !{i64 6699318081062747564, i64 844700110938769, !"foo"}
-!14 = !{i64 -2624081020897602054, i64 281563657672557, !"main"}
+!11 = !{i64 -2012135647395072713, i64 4294967295, i8 2, !"bar"}
+!12 = !{i64 9204417991963109735, i64 72617220756, i8 2, !"work"}
+!13 = !{i64 6699318081062747564, i64 844700110938769, i8 2, !"foo"}
+!14 = !{i64 -2624081020897602054, i64 281563657672557, i8 0, !"main"}
!18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
!47 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 15, type: !48, scopeLine: 15, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !50)
!48 = !DISubroutineType(types: !49)
diff --git a/llvm/test/CodeGen/X86/pseudo-probe-desc-check.ll b/llvm/test/CodeGen/X86/pseudo-probe-desc-check.ll
index 0c2bdfe9adcb1..9bb8997b96979 100644
--- a/llvm/test/CodeGen/X86/pseudo-probe-desc-check.ll
+++ b/llvm/test/CodeGen/X86/pseudo-probe-desc-check.ll
@@ -33,7 +33,7 @@ declare void @llvm.pseudoprobe(i64, i64, i32, i64)
!4 = !{i32 1, !"wchar_size", i32 4}
!5 = !{i32 7, !"uwtable", i32 2}
!6 = !{i32 7, !"frame-pointer", i32 2}
-!7 = !{i64 6028998432455395745, i64 281479271677951, !"extract1"}
+!7 = !{i64 6028998432455395745, i64 281479271677951, i8 0, !"extract1"}
!8 = distinct !DISubprogram(name: "extract1", scope: !1, file: !1, line: 4, type: !9, scopeLine: 4, spFlags: DISPFlagDefinition, unit: !0)
!9 = !DISubroutineType(types: !10)
!10 = !{}
diff --git a/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll b/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll
index f915aaccc06e1..58cebdcbb5ec8 100644
--- a/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll
+++ b/llvm/test/ThinLTO/X86/pseudo-probe-desc-import.ll
@@ -12,8 +12,8 @@
; RUN: llvm-lto -thinlto-action=import %t3.bc -thinlto-index=%t3.index.bc -o /dev/null 2>&1 | FileCheck %s --check-prefix=WARN
-; CHECK-NOT: {i64 6699318081062747564, i64 [[#]], !"foo"
-; CHECK: !{i64 -2624081020897602054, i64 [[#]], !"main"
+; CHECK-NOT: {i64 6699318081062747564, i64 [[#]], i8 [[#]], !"foo"
+; CHECK: !{i64 -2624081020897602054, i64 [[#]], i8 [[#]], !"main"
; WARN: warning: Pseudo-probe ignored: source module '{{.*}}' is compiled with -fpseudo-probe-for-profiling while destination module '{{.*}}' is not
diff --git a/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll b/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll
index 3ae0aea6e7573..e3d32b61f78bb 100644
--- a/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll
+++ b/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll
@@ -113,9 +113,9 @@ attributes #4 = { mustprogress nocallback nofree nosync nounwind willreturn memo
!5 = !{i32 7, !"frame-pointer", i32 2}
!6 = !{i32 1, !"EnableSplitLTOUnit", i32 0}
!7 = !{!"clang version 17.0.0"}
-!8 = !{i64 6699318081062747564, i64 4294967295, !"foo"}
-!9 = !{i64 7289175272376759421, i64 562954248388607, !"func"}
-!10 = !{i64 -2624081020897602054, i64 281582081721716, !"main"}
+!8 = !{i64 6699318081062747564, i64 4294967295, i8 0, !"foo"}
+!9 = !{i64 7289175272376759421, i64 562954248388607, i8 2, !"func"}
+!10 = !{i64 -2624081020897602054, i64 281582081721716, i8 0, !"main"}
!11 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 6, type: !12, scopeLine: 6, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
!12 = !DISubroutineType(types: !13)
!13 = !{}
diff --git a/llvm/test/Transforms/SampleProfile/csspgo-import-list-preinliner.ll b/llvm/test/Transforms/SampleProfile/csspgo-import-list-preinliner.ll
index 9e342f2b99da3..8e7d092230895 100644
--- a/llvm/test/Transforms/SampleProfile/csspgo-import-list-preinliner.ll
+++ b/llvm/test/Transforms/SampleProfile/csspgo-import-list-preinliner.ll
@@ -39,7 +39,7 @@ attributes #1 = { nocallback nofree nosync nounwind willreturn memory(inaccessib
!5 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !6)
!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
!7 = !{i32 2, !"Debug Info Version", i32 3}
-!8 = !{i64 -2624081020897602054, i64 563108639284859, !"main"}
+!8 = !{i64 -2624081020897602054, i64 563108639284859, i8 0, !"main"}
!9 = !DILocation(line: 11, column: 10, scope: !10)
!10 = !DILexicalBlockFile(scope: !11, file: !1, discriminator: 186646615)
!11 = distinct !DILexicalBlock(scope: !12, file: !1, line: 8, column: 40)
diff --git a/llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll b/llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll
index df56b55dcdf3c..83f513c3a614b 100644
--- a/llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll
+++ b/llvm/test/Transforms/SampleProfile/csspgo-profile-checksum-mismatch-attr.ll
@@ -52,9 +52,9 @@ attributes #0 = { "use-sample-profile" }
!5 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !6)
!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
!7 = !{i32 2, !"Debug Info Version", i32 3}
-!8 = !{i64 7546896869197086323, i64 4294967295, !"baz"}
-!9 = !{i64 -2012135647395072713, i64 281530612780802, !"bar"}
-!10 = !{i64 -2624081020897602054, i64 281582081721716, !"main"}
+!8 = !{i64 7546896869197086323, i64 4294967295, i8 0, !"baz"}
+!9 = !{i64 -2012135647395072713, i64 281530612780802, i8 0, !"bar"}
+!10 = !{i64 -2624081020897602054, i64 281582081721716, i8 0, !"main"}
!11 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13)
!12 = distinct !DISubroutineType(types: !13)
!13 = !{}
diff --git a/llvm/test/Transforms/SampleProfile/profile-correlation-irreducible-loops.ll b/llvm/test/Transforms/SampleProfile/profile-correlation-irreducible-loops.ll
index ef2fcc6a9e248..0682e499759ad 100644
--- a/llvm/test/Transforms/SampleProfile/profile-correlation-irreducible-loops.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-correlation-irreducible-loops.ll
@@ -177,8 +177,8 @@ attributes #0 = { noinline nounwind uwtable "use-sample-profile"}
attributes #1 = { nounwind }
!llvm.pseudo_probe_desc = !{!1079, !4496}
-!1079 = !{i64 -7702751003264189226, i64 158496288380146391, !"yyparse_1", null}
-!4496 = !{i64 7682762345278052905, i64 404850113186107133, !"foo1", null}
+!1079 = !{i64 -7702751003264189226, i64 158496288380146391, i8 2, !"yyparse_1", null}
+!4496 = !{i64 7682762345278052905, i64 404850113186107133, i8 0, !"foo1", null}
!132 = !{!"function_entry_count", i64 1}
!133 = !{!"branch_weights", i32 0, i32 86, i32 0}
!134 = !{!"branch_weights", i32 85, i32 9449, i32 1, i32 8212}
diff --git a/llvm/test/Transforms/SampleProfile/profile-inference-even-count-distribution.ll b/llvm/test/Transforms/SampleProfile/profile-inference-even-count-distribution.ll
index 407769a48701f..f6442c361879a 100644
--- a/llvm/test/Transforms/SampleProfile/profile-inference-even-count-distribution.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-inference-even-count-distribution.ll
@@ -167,7 +167,7 @@ attributes #4 = { inaccessiblememonly nounwind willreturn }
!llvm.pseudo_probe_desc = !{!7, !8, !9, !10}
-!7 = !{i64 7682762345278052905, i64 157181141624, !"foo1", null}
-!8 = !{i64 2494702099028631698, i64 208782362068, !"foo2", null}
-!9 = !{i64 -7908226060800700466, i64 189901498683, !"foo3", null}
-!10 = !{i64 -6882312132165544686, i64 241030178952, !"foo4", null}
+!7 = !{i64 7682762345278052905, i64 157181141624, i8 2, !"foo1", null}
+!8 = !{i64 2494702099028631698, i64 208782362068, i8 2, !"foo2", null}
+!9 = !{i64 -7908226060800700466, i64 189901498683, i8 2, !"foo3", null}
+!10 = !{i64 -6882312132165544686, i64 241030178952, i8 2, !"foo4", null}
diff --git a/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll b/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll
index a1858b50591da..dff611e43cf64 100644
--- a/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-inference-islands.ll
@@ -205,8 +205,8 @@ attributes #2 = { nounwind }
!llvm.pseudo_probe_desc = !{!7, !8}
-!7 = !{i64 -5646793257986063976, i64 120879332589, !"islands_1"}
-!8 = !{i64 -7683376842751444845, i64 69495280403, !"islands_2"}
-!9 = !{i64 -9095645063288297061, i64 156608410269, !"islands_3"}
+!7 = !{i64 -5646793257986063976, i64 120879332589, i8 2, !"islands_1"}
+!8 = !{i64 -7683376842751444845, i64 69495280403, i8 2, !"islands_2"}
+!9 = !{i64 -9095645063288297061, i64 156608410269, i8 2, !"islands_3"}
; CHECK-ENTRY-COUNT: = !{!"function_entry_count", i64 2}
diff --git a/llvm/test/Transforms/SampleProfile/profile-inference-rebalance-large.ll b/llvm/test/Transforms/SampleProfile/profile-inference-rebalance-large.ll
index cd14d7cc63bee..1fa3d86900bdd 100644
--- a/llvm/test/Transforms/SampleProfile/profile-inference-rebalance-large.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-inference-rebalance-large.ll
@@ -381,7 +381,7 @@ attributes #4 = { inaccessiblememonly nounwind willreturn }
!llvm.pseudo_probe_desc = !{!7, !8, !9, !10}
-!7 = !{i64 7682762345278052905, i64 157181141624, !"foo1", null}
-!8 = !{i64 2494702099028631698, i64 208782362068, !"foo2", null}
-!9 = !{i64 -7908226060800700466, i64 189901498683, !"foo3", null}
-!10 = !{i64 -6882312132165544686, i64 241030178952, !"foo4", null}
+!7 = !{i64 7682762345278052905, i64 157181141624, i8 2, !"foo1", null}
+!8 = !{i64 2494702099028631698, i64 208782362068, i8 2, !"foo2", null}
+!9 = !{i64 -7908226060800700466, i64 189901498683, i8 2, !"foo3", null}
+!10 = !{i64 -6882312132165544686, i64 241030178952, i8 2, !"foo4", null}
diff --git a/llvm/test/Transforms/SampleProfile/profile-inference-rebalance.ll b/llvm/test/Transforms/SampleProfile/profile-inference-rebalance.ll
index 3fa9769e17a44..2333848c33b30 100644
--- a/llvm/test/Transforms/SampleProfile/profile-inference-rebalance.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-inference-rebalance.ll
@@ -284,7 +284,7 @@ attributes #4 = { inaccessiblememonly nounwind willreturn }
!llvm.pseudo_probe_desc = !{!7, !8, !9, !10}
-!7 = !{i64 -5758218299531803684, i64 223598586707, !"countMultipliers", null}
-!8 = !{i64 2506109673213838996, i64 2235985, !"countMultipliers2", null}
-!9 = !{i64 -544905447084884130, i64 22985, !"countMultipliers3", null}
-!10 = !{i64 -2989539179265513123, i64 2298578, !"countMultipliers4", null}
+!7 = !{i64 -5758218299531803684, i64 223598586707, i8 2, !"countMultipliers", null}
+!8 = !{i64 2506109673213838996, i64 2235985, i8 2, !"countMultipliers2", null}
+!9 = !{i64 -544905447084884130, i64 22985, i8 2, !"countMultipliers3", null}
+!10 = !{i64 -2989539179265513123, i64 2298578, i8 2, !"countMultipliers4", null}
diff --git a/llvm/test/Transforms/SampleProfile/profile-inference.ll b/llvm/test/Transforms/SampleProfile/profile-inference.ll
index 5ee15a2ad5fe9..a973e7bdeeefa 100644
--- a/llvm/test/Transforms/SampleProfile/profile-inference.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-inference.ll
@@ -240,7 +240,7 @@ attributes #1 = { nounwind }
!llvm.pseudo_probe_desc = !{!6, !7, !8, !9}
-!6 = !{i64 7964825052912775246, i64 4294967295, !"test_1", null}
-!7 = !{i64 -6216829535442445639, i64 37753817093, !"test_2", null}
-!8 = !{i64 1649282507922421973, i64 69502983527, !"test_3", null}
-!9 = !{i64 -907520326213521421, i64 175862120757, !"sum_of_squares", null}
+!6 = !{i64 7964825052912775246, i64 4294967295, i8 2, !"test_1", null}
+!7 = !{i64 -6216829535442445639, i64 37753817093, i8 2, !"test_2", null}
+!8 = !{i64 1649282507922421973, i64 69502983527, i8 2, !"test_3", null}
+!9 = !{i64 -907520326213521421, i64 175862120757, i8 2, !"sum_of_squares", null}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll
index 0c7f3ac2a0d5e..64290f9d5aa2b 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-callee-profile-mismatch.ll
@@ -46,7 +46,7 @@ attributes #1 = { "profile-checksum-mismatch" "use-sample-profile" }
!10 = !DIFile(filename: "test2.c", directory: "/home/test", checksumkind: CSK_MD5, checksum: "553093afc026f9c73562eb3b0c5b7532")
!11 = !{i32 2, !"Debug Info Version", i32 3}
; Make a checksum mismatch in the pseudo_probe_desc
-!12 = !{i64 -2624081020897602054, i64 123456, !"main"}
+!12 = !{i64 -2624081020897602054, i64 123456, i8 0, !"main"}
!13 = !DILocation(line: 8, column: 10, scope: !14)
!14 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 186646591)
!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 7, column: 40)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-desc-attr.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-desc-attr.ll
new file mode 100644
index 0000000000000..f97b62424e48e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-desc-attr.ll
@@ -0,0 +1,48 @@
+
+; REQUIRES: x86-registered-target
+; RUN: opt < %s -passes=pseudo-probe -S -o - | FileCheck %s
+
+; CHECK: ![[#]] = !{i64 {{-?[0-9]+}}, i64 [[#]], i8 0, !"baz"}
+; CHECK: ![[#]] = !{i64 {{-?[0-9]+}}, i64 [[#]], i8 1, !"bar"}
+; CHECK: ![[#]] = !{i64 {{-?[0-9]+}}, i64 [[#]], i8 2, !"foo"}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @baz() #0 !dbg !11 {
+entry:
+ ret void
+}
+
+; Function Attrs: alwaysinline nounwind uwtable
+define dso_local void @bar() #1 !dbg !12 {
+entry:
+ ret void
+}
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local void @foo() #2 !dbg !13 {
+entry:
+ ret void
+}
+
+attributes #0 = { nounwind uwtable}
+attributes #1 = { alwaysinline nounwind uwtable}
+attributes #2 = { noinline nounwind uwtable}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 7, !"uwtable", i32 2}
+!6 = !{i32 7, !"frame-pointer", i32 2}
+!7 = !{!"clang version 17.0.0 "}
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !{}
+!11 = distinct !DISubprogram(name: "baz", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
+!12 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 10, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
+!13 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 15, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-desc-guid.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-desc-guid.ll
index b0040f445d9e0..3cbca82e2db29 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-desc-guid.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-desc-guid.ll
@@ -1,8 +1,8 @@
; REQUIRES: x86-registered-target
; RUN: opt < %s -passes=pseudo-probe -S -o - | FileCheck %s
-; CHECK: ![[#]] = !{i64 -3345296970173352005, i64 [[#]], !"foo.dbg"}
-; CHECK-NOT: ![[#]] = !{i64 6699318081062747564, i64 [[#]], !"foo"}
+; CHECK: ![[#]] = !{i64 -3345296970173352005, i64 [[#]], i8 [[#]], !"foo.dbg"}
+; CHECK-NOT: ![[#]] = !{i64 6699318081062747564, i64 [[#]], i8 [[#]], !"foo"}
; Function Attrs: noinline nounwind optnone uwtable
define dso_local void @foo() #0 !dbg !8 {
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
index 8306ddcc6a91d..96b27bff742b8 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
@@ -79,7 +79,12 @@ entry:
$foo3 = comdat any
-define void @foo3(i32 %x) comdat {
+define void @foo3(i32 %x) #0 comdat {
+entry:
+ ret void
+}
+
+define void @foo4(i32 %x) #1 {
entry:
ret void
}
@@ -87,6 +92,12 @@ entry:
; CHECK-IL: Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CHECK-IL-NEXT: declare void @llvm.pseudoprobe(i64, i64, i32, i64)
+; CHECK-IL: ![[#]] = !{i64 {{-?[0-9]+}}, i64 [[#]], i8 0, !"foo"}
+; CHECK-IL: ![[#]] = !{i64 {{-?[0-9]+}}, i64 [[#]], i8 0, !"foo2"}
+;; Function hash of foo3 and foo4 are fixed number 0xFFFFFFFF.
+; CHECK-IL: ![[#]] = !{i64 {{-?[0-9]+}}, i64 4294967295, i8 1, !"foo3"}
+; CHECK-IL: ![[#]] = !{i64 {{-?[0-9]+}}, i64 4294967295, i8 2, !"foo4"}
+
; CHECK-IL: ![[#FOO:]] = distinct !DISubprogram(name: "foo"
; CHECK-IL: ![[#FAKELINE]] = !DILocation(line: 0, scope: ![[#FOO]])
; CHECK-IL: ![[#REALLINE]] = !DILocation(line: 2, scope: ![[#DISC0:]])
@@ -104,13 +115,25 @@ entry:
; CHECK-ASM-ELF: .section .pseudo_probe_desc,"G", at progbits,.pseudo_probe_desc_foo,comdat
; CHECK-ASM-ELF-NEXT: .quad [[#GUID]]
; CHECK-ASM-ELF-NEXT: .quad [[#HASH:]]
-; CHECK-ASM-ELF-NEXT: .byte 3
+; CHECK-ASM-ELF-NEXT: .byte 3
; CHECK-ASM-ELF-NEXT: .ascii "foo"
; CHECK-ASM-ELF-NEXT: .section .pseudo_probe_desc,"G", at progbits,.pseudo_probe_desc_foo2,comdat
; CHECK-ASM-ELF-NEXT: .quad [[#GUID2]]
; CHECK-ASM-ELF-NEXT: .quad [[#HASH2:]]
; CHECK-ASM-ELF-NEXT: .byte 4
; CHECK-ASM-ELF-NEXT: .ascii "foo2"
+; CHECK-ASM-ELF-NEXT: .section .pseudo_probe_desc,"G", at progbits,.pseudo_probe_desc_foo3,comdat
+; CHECK-ASM-ELF-NEXT: .quad {{-?[0-9]+}}
+;; The hash is a fixed number 0x1FFFFFFFF
+; CHECK-ASM-ELF-NEXT: .quad 1152921508901814271
+; CHECK-ASM-ELF-NEXT: .byte 4
+; CHECK-ASM-ELF-NEXT: .ascii "foo3"
+; CHECK-ASM-ELF-NEXT: .section .pseudo_probe_desc,"G", at progbits,.pseudo_probe_desc_foo4,comdat
+; CHECK-ASM-ELF-NEXT: .quad {{-?[0-9]+}}
+;; The hash is a fixed number 0x2FFFFFFFF
+; CHECK-ASM-ELF-NEXT: .quad 2305843013508661247
+; CHECK-ASM-ELF-NEXT: .byte 4
+; CHECK-ASM-ELF-NEXT: .ascii "foo4"
; CHECK-ASM-COFF: .section .pseudo_probe_desc,"drD",same_contents,.pseudo_probe_desc_foo
; CHECK-ASM-COFF-NEXT: .globl .pseudo_probe_desc_foo
; CHECK-ASM-COFF-NEXT: .pseudo_probe_desc_foo:
@@ -125,42 +148,66 @@ entry:
; CHECK-ASM-COFF-NEXT: .quad [[#HASH2:]]
; CHECK-ASM-COFF-NEXT: .byte 4
; CHECK-ASM-COFF-NEXT: .ascii "foo2"
+; CHECK-ASM-COFF-NEXT: .section .pseudo_probe_desc,"drD",same_contents,.pseudo_probe_desc_foo3
+; CHECK-ASM-COFF-NEXT: .globl .pseudo_probe_desc_foo3
+; CHECK-ASM-COFF-NEXT: .pseudo_probe_desc_foo3:
+; CHECK-ASM-COFF-NEXT: .quad {{-?[0-9]+}}
+;; The hash is a fixed number 0x1FFFFFFFF
+; CHECK-ASM-COFF-NEXT: .quad 1152921508901814271
+; CHECK-ASM-COFF-NEXT: .byte 4
+; CHECK-ASM-COFF-NEXT: .ascii "foo3"
+; CHECK-ASM-COFF-NEXT: .section .pseudo_probe_desc,"drD",same_contents,.pseudo_probe_desc_foo4
+; CHECK-ASM-COFF-NEXT: .globl .pseudo_probe_desc_foo4
+; CHECK-ASM-COFF-NEXT: .pseudo_probe_desc_foo4:
+; CHECK-ASM-COFF-NEXT: .quad {{-?[0-9]+}}
+;; The hash is a fixed number 0x2FFFFFFFF
+; CHECK-ASM-COFF-NEXT: .quad 2305843013508661247
+; CHECK-ASM-COFF-NEXT: .byte 4
+; CHECK-ASM-COFF-NEXT: .ascii "foo4"
; CHECK-SEC-ELF: [Nr] Name Type {{.*}} ES Flg Lk Inf Al
; CHECK-SEC-ELF: [ 3] .text.foo PROGBITS {{.*}} 00 AX 0 0 16
; CHECK-SEC-ELF: [ 5] .text.foo2 PROGBITS {{.*}} 00 AX 0 0 16
; CHECK-SEC-ELF: [ 8] .text.foo3 PROGBITS {{.*}} 00 AXG 0 0 1
-; CHECK-SEC-ELF-COUNT-3: .pseudo_probe_desc PROGBITS
+; CHECK-SEC-ELF: [ 9] .text.foo4 PROGBITS {{.*}} 00 AX 0 0 16
+; CHECK-SEC-ELF-COUNT-4: .pseudo_probe_desc PROGBITS
; CHECK-SEC-ELF: .pseudo_probe PROGBITS {{.*}} 00 L 3 0 1
; CHECK-SEC-ELF-NEXT: .pseudo_probe PROGBITS {{.*}} 00 L 5 0 1
; CHECK-SEC-ELF-NEXT: .pseudo_probe PROGBITS {{.*}} 00 LG 8 0 1
+; CHECK-SEC-ELF-NEXT: .pseudo_probe PROGBITS {{.*}} 00 L 9 0 1
; CHECK-SEC-ELF-NOT: .rela.pseudo_probe
; CHECK-SEC-ELF: COMDAT group section [ 7] `.group' [foo3] contains 2 sections:
; CHECK-SEC-ELF-NEXT: [Index] Name
; CHECK-SEC-ELF-NEXT: [ 8] .text.foo3
-; CHECK-SEC-ELF-NEXT: [ 21] .pseudo_probe
+; CHECK-SEC-ELF-NEXT: [ 24] .pseudo_probe
+; CHECK-SEC-ELF-EMPTY:
+; CHECK-SEC-ELF-NEXT: COMDAT group section [ 11] `.group' [.pseudo_probe_desc_foo] contains 1 sections:
+; CHECK-SEC-ELF-NEXT: [Index] Name
+; CHECK-SEC-ELF-NEXT: [ 12] .pseudo_probe_desc
; CHECK-SEC-ELF-EMPTY:
-; CHECK-SEC-ELF-NEXT: COMDAT group section [ 10] `.group' [.pseudo_probe_desc_foo] contains 1 sections:
+; CHECK-SEC-ELF-NEXT: COMDAT group section [ 13] `.group' [.pseudo_probe_desc_foo2] contains 1 sections:
; CHECK-SEC-ELF-NEXT: [Index] Name
-; CHECK-SEC-ELF-NEXT: [ 11] .pseudo_probe_desc
+; CHECK-SEC-ELF-NEXT: [ 14] .pseudo_probe_desc
; CHECK-SEC-ELF-EMPTY:
-; CHECK-SEC-ELF-NEXT: COMDAT group section [ 12] `.group' [.pseudo_probe_desc_foo2] contains 1 sections:
+; CHECK-SEC-ELF-NEXT: COMDAT group section [ 15] `.group' [.pseudo_probe_desc_foo3] contains 1 sections:
; CHECK-SEC-ELF-NEXT: [Index] Name
-; CHECK-SEC-ELF-NEXT: [ 13] .pseudo_probe_desc
+; CHECK-SEC-ELF-NEXT: [ 16] .pseudo_probe_desc
; CHECK-SEC-ELF-EMPTY:
-; CHECK-SEC-ELF-NEXT: COMDAT group section [ 14] `.group' [.pseudo_probe_desc_foo3] contains 1 sections:
+; CHECK-SEC-ELF-NEXT: COMDAT group section [ 17] `.group' [.pseudo_probe_desc_foo4] contains 1 sections:
; CHECK-SEC-ELF-NEXT: [Index] Name
-; CHECK-SEC-ELF-NEXT: [ 15] .pseudo_probe_desc
+; CHECK-SEC-ELF-NEXT: [ 18] .pseudo_probe_desc
; CHECK-SEC2-ELF: [Nr] Name Type {{.*}} ES Flg Lk Inf Al
; CHECK-SEC2-ELF: [ 3] .text PROGBITS {{.*}} 00 AX 0 0 16
; CHECK-SEC2-ELF: [ 5] .text PROGBITS {{.*}} 00 AX 0 0 16
; CHECK-SEC2-ELF: [ 8] .text PROGBITS {{.*}} 00 AXG 0 0 1
-; CHECK-SEC2-ELF-COUNT-3: .pseudo_probe_desc PROGBITS
+; CHECK-SEC2-ELF: [ 9] .text PROGBITS {{.*}} 00 AX 0 0 16
+; CHECK-SEC2-ELF-COUNT-4: .pseudo_probe_desc PROGBITS
; CHECK-SEC2-ELF: .pseudo_probe PROGBITS {{.*}} 00 L 3 0 1
; CHECK-SEC2-ELF-NEXT: .pseudo_probe PROGBITS {{.*}} 00 L 5 0 1
; CHECK-SEC2-ELF-NEXT: .pseudo_probe PROGBITS {{.*}} 00 LG 8 0 1
+; CHECK-SEC2-ELF-NEXT: .pseudo_probe PROGBITS {{.*}} 00 L 9 0 1
; CHECK-SEC2-ELF-NOT: .rela.pseudo_probe
; CHECK-SEC-COFF-LABEL: Sections [
@@ -171,41 +218,57 @@ entry:
; CHECK-SEC-COFF: Number: 6
; CHECK-SEC-COFF-NEXT: Name: .text
; CHECK-SEC-COFF: Number: 7
+; CHECK-SEC-COFF-NEXT: Name: .text
+; CHECK-SEC-COFF: Number: 8
; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe_desc
; CHECK-SEC-COFF: Characteristics [
; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_LNK_COMDAT
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_READ
-; CHECK-SEC-COFF: Number: 8
+; CHECK-SEC-COFF: Number: 9
; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe_desc
; CHECK-SEC-COFF: Characteristics [
; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_LNK_COMDAT
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_READ
-; CHECK-SEC-COFF: Number: 9
+; CHECK-SEC-COFF: Number: 10
+; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe_desc
+; CHECK-SEC-COFF: Characteristics [
+; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
+; CHECK-SEC-COFF-NEXT: IMAGE_SCN_LNK_COMDAT
+; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE
+; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_READ
+; CHECK-SEC-COFF: Number: 11
; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe_desc
; CHECK-SEC-COFF: Characteristics [
; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_LNK_COMDAT
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_READ
-; CHECK-SEC-COFF: Number: 13
+; CHECK-SEC-COFF: Number: 15
; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe
; CHECK-SEC-COFF: Characteristics [
; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_LNK_COMDAT
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_READ
-; CHECK-SEC-COFF: Number: 14
+; CHECK-SEC-COFF: Number: 16
; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe
; CHECK-SEC-COFF: Characteristics [
; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_LNK_COMDAT
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE
; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_READ
-; CHECK-SEC-COFF: Number: 15
+; CHECK-SEC-COFF: Number: 17
+; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe
+; CHECK-SEC-COFF: Characteristics [
+; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
+; CHECK-SEC-COFF-NEXT: IMAGE_SCN_LNK_COMDAT
+; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_DISCARDABLE
+; CHECK-SEC-COFF-NEXT: IMAGE_SCN_MEM_READ
+; CHECK-SEC-COFF: Number: 18
; CHECK-SEC-COFF-NEXT: Name: .pseudo_probe
; CHECK-SEC-COFF: Characteristics [
; CHECK-SEC-COFF: IMAGE_SCN_CNT_INITIALIZED_DATA
@@ -223,36 +286,52 @@ entry:
; CHECK-SEC-COFF: Name: foo3
; CHECK-SEC-COFF: Section: .text (6)
; CHECK-SEC-COFF: }
+; CHECK-SEC-COFF: Name: foo4
+; CHECK-SEC-COFF: Section: .text (7)
+; CHECK-SEC-COFF: }
; CHECK-SEC-COFF: Name: .pseudo_probe_desc_foo
-; CHECK-SEC-COFF: Section: .pseudo_probe_desc (7)
+; CHECK-SEC-COFF: Section: .pseudo_probe_desc (8)
; CHECK-SEC-COFF: StorageClass: Static
; CHECK-SEC-COFF: }
; CHECK-SEC-COFF: Name: .pseudo_probe_desc_foo2
-; CHECK-SEC-COFF: Section: .pseudo_probe_desc (8)
+; CHECK-SEC-COFF: Section: .pseudo_probe_desc (9)
; CHECK-SEC-COFF: StorageClass: Static
; CHECK-SEC-COFF: }
; CHECK-SEC-COFF: Name: .pseudo_probe_desc_foo3
-; CHECK-SEC-COFF: Section: .pseudo_probe_desc (9)
+; CHECK-SEC-COFF: Section: .pseudo_probe_desc (10)
+; CHECK-SEC-COFF: StorageClass: Static
+; CHECK-SEC-COFF: }
+; CHECK-SEC-COFF: Name: .pseudo_probe_desc_foo4
+; CHECK-SEC-COFF: Section: .pseudo_probe_desc (11)
; CHECK-SEC-COFF: StorageClass: Static
; CHECK-SEC-COFF: }
; Section symbols
; CHECK-SEC-COFF: Name: .pseudo_probe
-; CHECK-SEC-COFF: Section: .pseudo_probe (13)
+; CHECK-SEC-COFF: Section: .pseudo_probe (15)
; CHECK-SEC-COFF: AuxSectionDef {
; CHECK-SEC-COFF: Selection: Associative
; CHECK-SEC-COFF-NEXT: AssocSection: .text (4)
; CHECK-SEC-COFF: }
; CHECK-SEC-COFF: Name: .pseudo_probe
-; CHECK-SEC-COFF: Section: .pseudo_probe (14)
+; CHECK-SEC-COFF: Section: .pseudo_probe (16)
; CHECK-SEC-COFF: AuxSectionDef {
; CHECK-SEC-COFF: Selection: Associative
; CHECK-SEC-COFF-NEXT: AssocSection: .text (5)
; CHECK-SEC-COFF: }
; CHECK-SEC-COFF: Name: .pseudo_probe
-; CHECK-SEC-COFF: Section: .pseudo_probe (15)
+; CHECK-SEC-COFF: Section: .pseudo_probe (17)
; CHECK-SEC-COFF: AuxSectionDef {
; CHECK-SEC-COFF: Selection: Associative
; CHECK-SEC-COFF-NEXT: AssocSection: .text (6)
+; CHECK-SEC-COFF: }
+; CHECK-SEC-COFF: Name: .pseudo_probe
+; CHECK-SEC-COFF: Section: .pseudo_probe (18)
+; CHECK-SEC-COFF: AuxSectionDef {
+; CHECK-SEC-COFF: Selection: Associative
+; CHECK-SEC-COFF-NEXT: AssocSection: .text (7)
+
+attributes #0 = { alwaysinline nounwind uwtable}
+attributes #1 = { noinline nounwind uwtable}
!llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!9, !10}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll
index 92f04d525b57f..7de179e91ed7e 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll
@@ -154,10 +154,10 @@ attributes #3 = { nocallback nofree nosync nounwind willreturn memory(inaccessib
!6 = !{i32 1, !"ThinLTO", i32 0}
!7 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
!8 = !{!"clang version 13.0.0 "}
-!9 = !{i64 -2012135647395072713, i64 4294967295, !"bar", null}
-!10 = !{i64 7546896869197086323, i64 4294967295, !"baz", null}
-!11 = !{i64 6699318081062747564, i64 281479271677951, !"foo", null}
-!12 = !{i64 -2624081020897602054, i64 563125815542069, !"main", null}
+!9 = !{i64 -2012135647395072713, i64 4294967295, i8 0, !"bar", null}
+!10 = !{i64 7546896869197086323, i64 4294967295, i8 0, !"baz", null}
+!11 = !{i64 6699318081062747564, i64 281479271677951, i8 0, !"foo", null}
+!12 = !{i64 -2624081020897602054, i64 563125815542069, i8 0, !"main", null}
!13 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !14, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17)
!14 = !DISubroutineType(types: !15)
!15 = !{!16, !16}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll
index b4083f26d9912..6305c0aa4c197 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-inline.ll
@@ -70,8 +70,8 @@ if.end:
}
; CHECK: !llvm.pseudo_probe_desc = !{![[#DESC0:]], ![[#DESC1:]]}
-; CHECK: ![[#DESC0]] = !{i64 [[#GUID1]], i64 [[#HASH1:]], !"foo"}
-; CHECK: ![[#DESC1]] = !{i64 [[#GUID2]], i64 [[#HASH2:]], !"zen"}
+; CHECK: ![[#DESC0]] = !{i64 [[#GUID1]], i64 [[#HASH1:]], i8 0, !"foo"}
+; CHECK: ![[#DESC1]] = !{i64 [[#GUID2]], i64 [[#HASH2:]], i8 0, !"zen"}
; CHECK: ![[PD1]] = !{!"branch_weights", i32 5, i32 0}
; CHECK: ![[PD2]] = !{!"branch_weights", i32 382915, i32 5}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-instsched.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-instsched.ll
index 852f8d2469785..64eae37af5a63 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-instsched.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-instsched.ll
@@ -30,5 +30,4 @@ attributes #1 = { inaccessiblememonly nounwind willreturn }
!llvm.pseudo_probe_desc = !{!0}
-!0 = !{i64 6699318081062747564, i64 4294967295, !"foo", null}
-
+!0 = !{i64 6699318081062747564, i64 4294967295, i8 0, !"foo", null}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll
index 632f14d5c2ffc..e29acfb5efc62 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-invoke.ll
@@ -96,7 +96,7 @@ entry:
ret void, !dbg !40
}
-; CHECK: ![[#]] = !{i64 -3270123626113159616, i64 4294967295, !"_Z3bazv"}
+; CHECK: ![[#]] = !{i64 -3270123626113159616, i64 4294967295, i8 2, !"_Z3bazv"}
attributes #0 = { mustprogress noinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
attributes #1 = { noinline noreturn nounwind uwtable "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-missing-probe.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-missing-probe.ll
index ff64e5bde2f9f..2f8157252d839 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-missing-probe.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-missing-probe.ll
@@ -163,9 +163,9 @@ attributes #3 = { mustprogress nocallback nofree nosync nounwind willreturn memo
!12 = !{i32 7, !"uwtable", i32 2}
!13 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
!14 = !{!"clang version 20.0.0"}
-!15 = !{i64 -2012135647395072713, i64 4294967295, !"bar"}
-!16 = !{i64 7546896869197086323, i64 191430930410, !"baz"}
-!17 = !{i64 -2624081020897602054, i64 563091374530180, !"main"}
+!15 = !{i64 -2012135647395072713, i64 4294967295, i8 2, !"bar"}
+!16 = !{i64 7546896869197086323, i64 191430930410, i8 0, !"baz"}
+!17 = !{i64 -2624081020897602054, i64 563091374530180, i8 0, !"main"}
!18 = distinct !DISubprogram(name: "bar", scope: !3, file: !3, line: 3, type: !19, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !21)
!19 = !DISubroutineType(types: !20)
!20 = !{null, !6}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata-2.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata-2.ll
index 379dcfcab338d..39db6f13a4b6c 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata-2.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata-2.ll
@@ -46,7 +46,7 @@ attributes #0 = {"use-sample-profile"}
!0 = !{i32 7, !"Dwarf Version", i32 4}
!1 = !{i32 2, !"Debug Info Version", i32 3}
-!2 = !{i64 6699318081062747564, i64 563022570642068, !"foo", null}
+!2 = !{i64 6699318081062747564, i64 563022570642068, i8 0, !"foo", null}
!4 = distinct !DISubprogram(name: "foo", scope: !5, file: !5, line: 9, type: !6, scopeLine: 9, spFlags: DISPFlagDefinition, unit: !9)
!5 = !DIFile(filename: "test.cpp", directory: "test")
!6 = !DISubroutineType(types: !7)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata.ll
index d7a58b4cbbcf6..7da01f3cfe933 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-metadata.ll
@@ -47,7 +47,7 @@ exit:
attributes #0 = {"use-sample-profile"}
declare void @llvm.pseudoprobe(i64, i64, i32, i64) #1
!llvm.pseudo_probe_desc = !{!4496}
-!4496 = !{i64 6699318081062747564, i64 158517001042, !"foo", null}
+!4496 = !{i64 6699318081062747564, i64 158517001042, i8 0, !"foo", null}
; CHECK: ![[ENTRY_PROF]] = !{!"branch_weights", i32 10, i32 6}
; CHECK: ![[SWITCH_PROF]] = !{!"branch_weights", i32 1, i32 9536, i32 1}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
index e1d717c63e9ed..ba429ee52530f 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
@@ -194,10 +194,10 @@ attributes #5 = { nocallback nofree nosync nounwind readnone speculatable willre
!9 = !{i32 1, !"wchar_size", i32 4}
!10 = !{i32 7, !"uwtable", i32 2}
!11 = !{!""}
-!12 = !{i64 6699318081062747564, i64 4294967295, !"foo"}
-!13 = !{i64 -2012135647395072713, i64 4294967295, !"bar"}
-!14 = !{i64 -5844448289301669773, i64 4294967295, !"matched"}
-!15 = !{i64 -2624081020897602054, i64 844635331715433, !"main"}
+!12 = !{i64 6699318081062747564, i64 4294967295, i8 0, !"foo"}
+!13 = !{i64 -2012135647395072713, i64 4294967295, i8 2, !"bar"}
+!14 = !{i64 -5844448289301669773, i64 4294967295, i8 2, !"matched"}
+!15 = !{i64 -2624081020897602054, i64 844635331715433, i8 0, !"main"}
!16 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !19)
!17 = !DISubroutineType(types: !18)
!18 = !{!6, !6}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
index cda66b1ecf6d2..669f499e151f4 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
@@ -144,8 +144,8 @@ attributes #4 = { mustprogress nocallback nofree nosync nounwind willreturn }
!7 = !{i32 7, !"uwtable", i32 2}
!8 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
!9 = !{!"clang version 19.0.0"}
-!10 = !{i64 -4364451034228175269, i64 1970329131941887, !"test_direct_call"}
-!11 = !{i64 -8563147518712133441, i64 1688922477484692, !"test_indirect_call"}
+!10 = !{i64 -4364451034228175269, i64 1970329131941887, i8 0, !"test_direct_call"}
+!11 = !{i64 -8563147518712133441, i64 1688922477484692, i8 0, !"test_indirect_call"}
!12 = distinct !DISubprogram(name: "test_direct_call", scope: !1, file: !1, line: 10, type: !13, scopeLine: 10, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
!13 = !DISubroutineType(types: !14)
!14 = !{!15, !15}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll
index f58dc75490194..b720b10a7b8ad 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-lto.ll
@@ -160,8 +160,8 @@ attributes #4 = { nounwind }
!44 = !{i32 999990, i64 59, i32 16}
!45 = !{i32 999999, i64 59, i32 16}
!46 = !{!"clang version 18.0.0"}
-!47 = !{i64 6699318081062747564, i64 563022570642068, !"foo"}
-!48 = !{i64 -2624081020897602054, i64 1126124211449298, !"main"}
+!47 = !{i64 6699318081062747564, i64 563022570642068, i8 0, !"foo"}
+!48 = !{i64 -2624081020897602054, i64 1126124211449298, i8 0, !"main"}
!49 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 11, type: !50, scopeLine: 12, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !52)
!50 = !DISubroutineType(types: !51)
!51 = !{!6}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
index 02c8a8cd6f827..10540e3163a02 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
@@ -253,9 +253,9 @@ attributes #6 = { nocallback nofree nosync nounwind speculatable willreturn memo
!10 = !{i32 7, !"uwtable", i32 2}
!11 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
!12 = !{!"clang version 17.0.0"}
-!13 = !{i64 -2012135647395072713, i64 4294967295, !"bar"}
-!14 = !{i64 6699318081062747564, i64 563022570642068, !"foo"}
-!15 = !{i64 -2624081020897602054, i64 1126158552146340, !"main"}
+!13 = !{i64 -2012135647395072713, i64 4294967295, i8 2, !"bar"}
+!14 = !{i64 6699318081062747564, i64 563022570642068, i8 1, !"foo"}
+!15 = !{i64 -2624081020897602054, i64 1126158552146340, i8 0, !"main"}
!16 = distinct !DISubprogram(name: "bar", scope: !3, file: !3, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !19)
!17 = !DISubroutineType(types: !18)
!18 = !{!6, !6}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll
index dbf3dda46ee28..92dfecf979e6b 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll
@@ -90,8 +90,8 @@ attributes #3 = { mustprogress nocallback nofree nosync nounwind willreturn memo
!12 = !{i32 7, !"uwtable", i32 2}
!13 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
!14 = !{!"clang version 21.0.0git (https://github.com/llvm/llvm-project.git c9f1d2cbf18990311ea1287cc154e3784a10a3b0)"}
-!15 = !{i64 5326982120444056491, i64 4294967295, !"_Z3fool"}
-!16 = !{i64 -2624081020897602054, i64 281582264815352, !"main"}
+!15 = !{i64 5326982120444056491, i64 4294967295, i8 2, !"_Z3fool"}
+!16 = !{i64 -2624081020897602054, i64 281582264815352, i8 0, !"main"}
!17 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fool", scope: !3, file: !3, line: 3, type: !18, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !21)
!18 = !DISubroutineType(types: !19)
!19 = !{null, !20}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll
index 69e9bce2ceb0e..a7f2c11043ad3 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll
@@ -102,9 +102,9 @@ attributes #3 = { mustprogress nocallback nofree nosync nounwind willreturn memo
!12 = !{i32 7, !"uwtable", i32 2}
!13 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
!14 = !{!"clang version 19.0.0"}
-!15 = !{i64 8236371237083957767, i64 4294967295, !"bar_new"}
-!16 = !{i64 -837213161392124280, i64 281479271677951, !"foo_new"}
-!17 = !{i64 -2624081020897602054, i64 281582264815352, !"main"}
+!15 = !{i64 8236371237083957767, i64 4294967295, i8 0, !"bar_new"}
+!16 = !{i64 -837213161392124280, i64 281479271677951, i8 0, !"foo_new"}
+!17 = !{i64 -2624081020897602054, i64 281582264815352, i8 0, !"main"}
!18 = distinct !DISubprogram(name: "bar_new", scope: !3, file: !3, line: 3, type: !19, scopeLine: 3, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
!19 = !DISubroutineType(types: !20)
!20 = !{null}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll
index 426c761f06476..9f5ff67cff96f 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll
@@ -233,13 +233,13 @@ attributes #4 = { mustprogress nocallback nofree nosync nounwind willreturn memo
!12 = !{i32 7, !"uwtable", i32 2}
!13 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
!14 = !{!"clang version 19.0.0git (https://github.com/llvm/llvm-project.git 2e1509152224d8ffbeac84c489920dcbaeefc2b2)"}
-!15 = !{i64 -2012135647395072713, i64 4294967295, !"bar"}
-!16 = !{i64 2964250471062803127, i64 206551239323, !"new_block_only"}
-!17 = !{i64 7546896869197086323, i64 281479271677951, !"baz"}
-!18 = !{i64 5381804724291869009, i64 844429225099263, !"new_foo"}
-!19 = !{i64 -5610330892148506720, i64 281479271677951, !"test_noninline"}
-!20 = !{i64 2711072140522378707, i64 281479271677951, !"cold_func"}
-!21 = !{i64 -2624081020897602054, i64 1126003093360596, !"main"}
+!15 = !{i64 -2012135647395072713, i64 4294967295, i8 2, !"bar"}
+!16 = !{i64 2964250471062803127, i64 206551239323, i8 0, !"new_block_only"}
+!17 = !{i64 7546896869197086323, i64 281479271677951, i8 0, !"baz"}
+!18 = !{i64 5381804724291869009, i64 844429225099263, i8 0, !"new_foo"}
+!19 = !{i64 -5610330892148506720, i64 281479271677951, i8 2, !"test_noninline"}
+!20 = !{i64 2711072140522378707, i64 281479271677951, i8 0, !"cold_func"}
+!21 = !{i64 -2624081020897602054, i64 1126003093360596, i8 0, !"main"}
!22 = distinct !DISubprogram(name: "bar", scope: !3, file: !3, line: 3, type: !23, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !25)
!23 = !DISubroutineType(types: !24)
!24 = !{!6, !6}
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll
index 15a19477f7509..69754246fc82f 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll
@@ -114,9 +114,9 @@ attributes #3 = { mustprogress nocallback nofree nosync nounwind willreturn memo
!12 = !{i32 7, !"uwtable", i32 2}
!13 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
!14 = !{!"clang version 20.0.0"}
-!15 = !{i64 -2012135647395072713, i64 4294967295, !"bar"}
-!16 = !{i64 -2115950948644264162, i64 281479271677951, !"foo_rename"}
-!17 = !{i64 -2624081020897602054, i64 281582264815352, !"main"}
+!15 = !{i64 -2012135647395072713, i64 4294967295, i8 2, !"bar"}
+!16 = !{i64 -2115950948644264162, i64 281479271677951, i8 2, !"foo_rename"}
+!17 = !{i64 -2624081020897602054, i64 281582264815352, i8 0, !"main"}
!18 = distinct !DISubprogram(name: "bar", scope: !3, file: !3, line: 3, type: !19, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !21)
!19 = !DISubroutineType(types: !20)
!20 = !{!6, !6}
>From 08702c42057cca80b4a1c1ae8fe1528863963ccc Mon Sep 17 00:00:00 2001
From: Wei Wang <apollo.mobility at gmail.com>
Date: Mon, 13 Apr 2026 21:41:25 -0700
Subject: [PATCH 2/2] [CSPreInliner] Use always_inline/noinline attributes in
pre-inliner
Teach the CSSPGO pre-inliner to respect always_inline and noinline
attributes encoded in pseudo probe descriptors:
- always_inline callees are unconditionally inlined and don't consume
the caller's size budget.
- noinline callees are never inlined.
Attributes are stamped onto FunctionSamples' ContextAttributeMask
during CSPreInliner construction by iterating only the small set of
attributed probe descriptors, avoiding per-candidate MD5 hashing.
Attributes are also propagated through mergeContextNode to survive
context profile promotion.
---
llvm/include/llvm/ProfileData/SampleProf.h | 2 +
.../Transforms/IPO/SampleContextTracker.cpp | 4 ++
.../Inputs/cs-preinline-attr.perfbin | Bin 0 -> 20368 bytes
.../tools/llvm-profgen/cs-preinline-attr.test | 52 ++++++++++++++++++
llvm/tools/llvm-profgen/CSPreInliner.cpp | 46 +++++++++++++++-
llvm/tools/llvm-profgen/ProfiledBinary.h | 3 +
6 files changed, 106 insertions(+), 1 deletion(-)
create mode 100755 llvm/test/tools/llvm-profgen/Inputs/cs-preinline-attr.perfbin
create mode 100644 llvm/test/tools/llvm-profgen/cs-preinline-attr.test
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 956d411bfba03..9882170b203c5 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -490,6 +490,8 @@ enum ContextAttributeMask {
ContextShouldBeInlined = 0x2, // Leaf of context should be inlined
ContextDuplicatedIntoBase =
0x4, // Leaf of context is duplicated into the base profile
+ ContextAlwaysInline = 0x8, // Function has always_inline attribute
+ ContextNoInline = 0x10, // Function has noinline attribute
};
// Represents a context frame with profile function and line location
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
index c8f203e8ce3ee..6b78264ec3dd1 100644
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -561,6 +561,10 @@ void SampleContextTracker::mergeContextNode(ContextTrieNode &FromNode,
FromSamples->getContext().setState(MergedContext);
if (FromSamples->getContext().hasAttribute(ContextShouldBeInlined))
ToSamples->getContext().setAttribute(ContextShouldBeInlined);
+ if (FromSamples->getContext().hasAttribute(ContextAlwaysInline))
+ ToSamples->getContext().setAttribute(ContextAlwaysInline);
+ if (FromSamples->getContext().hasAttribute(ContextNoInline))
+ ToSamples->getContext().setAttribute(ContextNoInline);
} else if (FromSamples) {
// Transfer FromSamples from FromNode to ToNode
ToNode.setFunctionSamples(FromSamples);
diff --git a/llvm/test/tools/llvm-profgen/Inputs/cs-preinline-attr.perfbin b/llvm/test/tools/llvm-profgen/Inputs/cs-preinline-attr.perfbin
new file mode 100755
index 0000000000000000000000000000000000000000..b27f0ee5760835e1e85e19e21d8bfafa2d9323fe
GIT binary patch
literal 20368
zcmeHPYj7Labw0ZbfDecyD2fqn%F<e~Y$*-_f-h6FEddlMfwd&7A}P|7*h at T?BvKGS
zfu%{Rsmw%5D%zH#wBw9tnpW-fk2{l0nzWN?GUHZ`Y{!{SRNHkEx9&sf#H~7RQ+3j$
ztTPpQ&h9<HVhO5lr_=n%-r?PI&vzd8?%9jIkKN at Hqel<Rk_4P&H~_?{frtrNM}XHx
zO;%uo&;${<2kr(RA|A{UQ$o}#F?~z61WfCNo`RI at I;ddMNEO{-asW)Zg+$4&k|LsL
z(R#1NGKGM}lbtCNJc#pS`fX7GQ at fp6pKAF5HDVP6?G|=S?RFHKNYGM`9I)(|UK4iL
z1!X%57ulT<b|-`#(~Cl%DaVuANO at 4ib5MyON4!N<Vlb)Z5x{9TE$pU+K2z3b%CR4R
z$4LEe>N_Fq0>WWH#F6QUXdhFq at 8{8u;_`kIU*?mdzRj+~Wy$i$RIm`rq!R;up-ieP
zlg=(KcP$SNbPe<ci at 9KrX+Z6wI;c+_IX(&UDqJQ^gwebfBqx6)r$jQkzLW^(Yy$7|
zKsH-z=<Y>Kt$|Y<YvJRY;Gk)E=1Hw+#0!SD5Km`;cslU`IC6As_>k5U?Ck?>Y~q-f
z(hK at rx@hQyiDQQ{xvV}BPh at m-F}IM*3KyECT2m3*P>F{GZOHR=`-9JZ4llP1cOXXT
z+QEoPsSi*LI9<2l?2BYKY&c&N!q;s$_btNR<VL`afExif0&WD{2)Ge&Bj85hXCVSV
zY}xr<?Cd|)$DZ<jF9Z<#)MZ01y%szBQvDV4^hqB6=drcdAB(O1UF_`lZ=9GI4POp_
zEB4KA&a_6L)c)&e^>4GmmR;w{`d2FK#n^=hzeav9c)vnq|GLqRRb at q04W(->yH?5Y
zilC^!X6pBUlE}WBv9%kq7k_Xt_Tst{lU|9ve$&{74jvQ^>Py#VTXx|hhwZ7os}GK%
z7#7<mV{7kRKV2%7Uh(!KB0YNL;+5G-<63q_fb`#evVyIxD_#%kvcI+UmM?jKh>P(n
zRy}L$_j<oitZ47}g&hZ#JKjL#`s-Mc^wz!Jmy!7h%O|5#(Noch at fT=u5`KF${NvGZ
z$<~IEr}jLH`fGoC{a4ZasiyPDj7aMv(zSiBj$U}cSGscc<(tv7ugK9)++01k6Be)T
zyZ({rN28BLwMXd+m9Jm<iXrGGHv(=1+z7Z4a3kPGz>R<#0XG6}1l$O?5pW~mh=5dn
z0%r-NPUU`jW>|+gaHCY3K^j47AccP|mChpxApJa273uFG?MC_%(i2FpA*Hm<A|RbU
z3DR;v>fBOSPtUCB`Q8r9&;LuQbT{dY)CCCZ#{3du;_<zyNA1XC-3>@7k=``s3}TH0
zU?MDodETbif&41w6k at j_g=ll&tUT3B&%6gQQ`<@ZQKTdrZtfUvR)?GSMw`3O`kwYa
z<N1v8X<0tjyw`+L7>Jt2r0*s-0&WD{2)Ge&Bj85Bjer{gHv(=1+z9-vMgSiIQ+iFj
zPtEUU(z~3LwpaugY7w*4B>2n1&Rm}%#_vUU3OT=@e7oRd2O?&P){c}|{-c|v9P!Ia
zgoyQyH`0J7Lag~NHPPpUoY##D1Y+`A1Zd4jsof&wBdvoeJum!my)z~Xz-v>szge={
zb6)7<LtV3d92Z=6n*13d7jAxPgx at Q#)qoU&M-CktQupE;%=^@EIM^TTR(ra8`nvB6
z?^pMp)Kh9KZU{kFxNjdMGx6-4`nX;wrgK at fr#ILg?4C;->fU+7$QOq~A!Ozk6TxI|
zAr#N&GMU`cQYe#oe4#5}$bC#t8bQ*I_v-`w1L4`;p1y(PV5&EzcMl|H_1 at WVav%{-
z%-%OJm>38rlKUWfYN#(C?%6*GCr=F(7PHwjRzH_aIvWear-tI$Y%Y<`))XI^8tPu|
zP4#s555(1AFjQPAru2L<gs**v;)!%uHfQL;B#fUJ>V@$`Lt!{{cxdnziieL5b%k#s
zKN3YBC&tLz=ukJL$MI0rhIBS<V1;HiqHOsTOqDXetZk%|T?IXpOR|vzvy2_!nKwTc
zJM~+K9>Pvq!fPF9k7t(RD at 84x#h2dol$MNVGP?fz!O;1Y-qa<brM6^qmV%JK`JFHB
z4?QsPqL4#VHdk1PXN2?%FLb<p^c(9RwWJCtv$@=Re}3T~22#(yz5dMKyw`!Rb9?AL
za|Y`qX|-Nn1<z`SZ<XGwS5|xKR>8}TWLa80u6(jV=GIFftuA at E6~uvESNd;s?S?X+
zD|xA^fL`CA-#+a((F^EDqIXHX at o@P&nZx4#R!e76$e7mB;rrtUZsCXTnXig_XDvT7
z!rpfaofm|RX)Ql2V_Hjx+uC`HHgj9QE$Upv*m*%BcNoSyL<t!gzKBP{X~>Yx8>=*x
z#<C(!{vs0aNObd-rW&E)G#0$lq}M8eEYodRn)*{bqV{)@fKLGrsnvNc3iTeZM^<RO
zk*2<i#|HQ#JT^+N;L%U at czk$(w#MzCc5<Ct?nlTAJ|DCSXC`f`gRR7auO6C&z@*UZ
zqg_R5$`7{@-Xhn*?NkWVJ&4CvX?zQG67Q!b)a}9p>}U>|HIAF at QOfP9+e%Vy&$t-_
zKb>N|+0gdJdn_~abn&YYYHEF(G$|Lcu&tp;Lyjc>c0)VW2U4rFXHT0~*;Cja*xtFP
zwR5YlxczBqJH|RF5f|w=eOtH<1Ly_0g9 at GsuIw4|-|xTYwz0N at 7(g#+SkM^i`GS&N
zuHK0iW7I1O^>-n2!0*SgwgF9dlk>(tQSjd(h1;n9Y5ybsDgRf8@#MC|;EA?h#;HJ4
zpUq9PvprU8TF6tSvVh+_8;mvLwpMP9NtVyHF~rZ{c)g9BlMgyK8G-hGp!0!0!|cVb
zcprsLl77<MM8piI3GAXGcC=k`i+88=A!2^F^Np)bO=l<CXNoTu7he7o51y(VXpN8z
z+P%$<A+Ogriyi6HqHJJ!$%cQ*Ti1A(w^_+Q;B5q-=hEuxUr0*Jtha6&E&X#|_%)w0
zSO0U~fUoh(b`1j&<+S1bpigO<1=-{Aecihqsq72VW%9`13H$s({C!B$|M%D&G8Xco
zWD!Sp-5M{t;)YS^GV~&iY at zwwf*xAJ>B5qp4&lHR&x989MWdj{7pi6iP;*d%bk=}n
zz`+CxP&MEH4RShSRf80mpE9Lk0<Q;MTfR1`xJKQAMg$!Z^kG3~1YH#L89{eMB6NxW
z2d8uEu>3pvqV{12GRK;oNbkl>zkW~mL21%+aLn^N(&Q!Zgol(?Ksv1x_2}m$N%NeO
zenq<P2?)?|+j4u63*oyy&c)sNa;X(#Cp35L-i at yGCDYZ?nQdjuirpp8nRC+48iI<$
zOJ at E3x6~;MowX0*De~i=JeHrq{D+vYVtxbjo0#9i{Fj)oWBvi=o0t{z=ZJ;Pl7b^m
zhZ5N!R&baZtcfcyT@&||*BN$a3O1Y97uC40ybh_x>!8;BU%~6n>xOFmhVuHO8pi=q
zjJ?&kzdSFg#<u{^A*yj)m8lUsWR?|Z2L3M7E>xferfcd4%Imml{Z`=hTQ$BFYOOyM
zXe)p3SFMjv6~*tZ8gGY=n)nCsv?6v at v#bD)dNq<#UIkVQ72s8>6TiJ05S#*?uy-c1
zL8g?yX08?~<*yu_IQ{xJCI!rS!v=u@>IRGe1 at 7Dg-@OU`!40^ChzB14I|T<2kgx{+
zy{JMVhP0OcIN~y{x_O=|)PcT`CH$hJKhF!C_ZL|I3>GA)TwiJ*r6(~<u(7{=8FBIt
z;=WC~lwQCr;hmQEA50Jre}%bL`!_qUADMny;KO%DuvDFD`JtPZ1RMMR7V|_l{_beb
zA<<KHM}O`|+?*#?R|zK(R~;C!pEdD~zn3iHt+ at Na;~9-VlnlfJh&z8rzl8W!dE@=$
z6NF=&c|7I)k<SP}&fn?hg+7nRtp63nRkH=vhZhlVm7AbmjZ_J)5|00sfbV16{=Y_?
z+7l7ioqx&SLp*@}XRjlke^2_dA5MsV$OF=|!jCF&8j~sA6n>oXX_T#cXGFc`y9TJ=
zf_SUEF`k`>tK`s;`h_1=_ at N~brBTA=8u$OuVzEFUuZM6KgtjlVfj-<ANX^F$>r3`f
zI2^(aj`AO>;&<-Jf)Nhp at XKx7?9hz(93W6M7V$%cBviig){F%$N#A)FQ8Sg(<}$ei
zF7#7Iu29tCi_3u9I{A!l=&4|LZ4GS}_b;?~p%7ou^sG@>f!RWQLDy1?3kxe~V&iZZ
zB5hQuZaSCLw8JN($40f$<0Be=Z$9$K@#wL!L+B3ot+dgYkj6$%f_CKS!^6>|+QWwr
zkB?4h6Vc(LqiEs$<~{O0zjU{znwp-98*wnt%`q3nFrmo$6n>M5tn+JmEmh2E^YLto
zzN8;}7;RJOthQLxQ|4?J!<GUqmC?-I9-QftbKL#W@@e|=K2a=++RO>Brs0?Q92fdz
zzvg%N&bhJ~dVpYYWx<FikQxPx&T|Q8-+CbrL33YhF1r}S?Yq2QFjfQ<9UkkVZWJWh
ze7rah!PH6?{aMr~SQ4>OSjA{4!*h$10R*WNK+w>a at ko6KWjQn2L498ImiZJ~FlSj<
z-Nv%uGP;W|q?1 at z4t?Pvn9pFra)x08+LT<tP6<J=Q)r$Y;`yS!h`k+8F at Zf*FQN}S
zWVJrAn9ihvw8BXz7IBS(p7q4y9ENf>$K`k;QP3Y}j`rBNjB({+JVobY8IuZL&I+GP
zX7K+ll%1nU%gxRfb%g$J679d^Ee*#?D|ILlv%vqSh4<fu*wp1RQJOi>o;lkO3VZ&)
zBr3}6sou_Dc>uHBp63<(|4CGbKHujV9>KzoagUwrUl#Q*E0zYgpYQ9=`j4X;JriL2
z=Y;)p!hYFuV?I}qu*~)y0zZY6RN0=NLj*)a`MHBo-w?mYP=HIfa{GCma7Nhk{h!;<
z^)USfw5Mkf%=3EUys%eA#k7TFZ at ++n_KHZA?VtBVOwsege#Yb~3pMBYpF=tMCI4pP
z`JB0N3OF;aPQq;&wB~OWdme99VgDQC+?E7fMlNE>UjJn=ZeDJ+RB(n+m7Mi|4y9zz
z=g;#AP%SfOeSH3w6yP_pV7KS>Q$W~{u%e*O`oG|?=XKV!u%`vEJvr;AZN+BHqPv#o
zwfz5Yochc&{RY~$*z9 at T77HSXiS;b5!ZUsq1$6DXeqP`4|1;wMTg3f??KrOAMO%s=
z+w=c at xfVhYlWHC-2exNQ^LD#^dA{6j+3 at d_J+K|qcO3RSU+oa~tjG4w>-R1SNQc|c
z^FnnKduppa-9Uygp<WiT{iQNu71*BXe;{1T{*Q&j1FR_gFw3+a3tYBW^`BvjyI=>g
z?9}ZP_NxxPCwnY_+fVmcN>2NI2=FsklJh$H!@b1e>%tBLXZ#O}`Yr9sG2dsER~+^Z
z79QHf{vZ1-`+FS*4i+BS#C{uXCS#)hZBI^nUD!LX|GUE8>o8!JX<pdd>!9+DK?|97
z2-uF{j|+Qe{jvR)MvN5&WqYPi3wwteTom>{aR}IsVOkG5F_JwsWNAFWih??Ii7(%~
zZMv at t`+Cv2_MnU|DgU1NzCi0w{_Wb#;kBs6J>+oite=LNTK1jd#{VUUfrEwr0y%)C
A<^TWy
literal 0
HcmV?d00001
diff --git a/llvm/test/tools/llvm-profgen/cs-preinline-attr.test b/llvm/test/tools/llvm-profgen/cs-preinline-attr.test
new file mode 100644
index 0000000000000..97c1c759ca498
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/cs-preinline-attr.test
@@ -0,0 +1,52 @@
+; Test that always_inline/noinline attributes are correctly decoded from pseudo
+; probe descriptors.
+
+; RUN: llvm-profgen --show-disassembly-only --show-pseudo-probe --binary=%S/Inputs/cs-preinline-attr.perfbin --output=/dev/null 2>&1 | FileCheck %s
+
+; CHECK: Name: always_inlined_callee
+; CHECK-NEXT: Hash: {{[0-9]+}} Attributes: 1
+; CHECK: Name: foo
+; CHECK-NEXT: Hash: {{[0-9]+}} Attributes: 0
+; CHECK: Name: noinline_callee
+; CHECK-NEXT: Hash: {{[0-9]+}} Attributes: 2
+; CHECK: Name: normal_callee
+; CHECK-NEXT: Hash: {{[0-9]+}} Attributes: 0
+; CHECK: Name: main
+; CHECK-NEXT: Hash: {{[0-9]+}} Attributes: 0
+
+; The binary is built from the source below with:
+; clang -O2 -g -fpseudo-probe-for-profiling -fdebug-info-for-profiling \
+; cs-preinline-attr-test.c -o cs-preinline-attr.perfbin \
+; --target=x86_64-linux-gnu -fno-pie -no-pie
+;
+; volatile int state = 100;
+;
+; __attribute__((always_inline))
+; int always_inlined_callee(int x) {
+; return x + state;
+; }
+;
+; __attribute__((noinline))
+; int noinline_callee(int x) {
+; return x * state;
+; }
+;
+; int normal_callee(int x) {
+; return x - state;
+; }
+;
+; int foo(int x) {
+; int r = 0;
+; r += always_inlined_callee(x);
+; r += noinline_callee(x);
+; r += normal_callee(x);
+; return r;
+; }
+;
+; int main() {
+; int r = 0;
+; for (int i = 0; i < 1000000; i++) {
+; r += foo(i);
+; }
+; return r;
+; }
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index 87df6996aa435..1817140a0b1ea 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -31,6 +31,10 @@ STATISTIC(PreInlNumCSInlinedHitMaxLimit,
STATISTIC(
PreInlNumCSInlinedHitGrowthLimit,
"Number of functions with FDO inline stopped due to growth size limit");
+STATISTIC(PreInlNumCSInlinedAlwaysInline,
+ "Number of functions inlined due to always_inline attribute");
+STATISTIC(PreInlNumCSNotInlinedNoInline,
+ "Number of functions not inlined due to noinline attribute");
// The switches specify inline thresholds used in SampleProfileLoader inlining.
// TODO: the actual threshold to be tuned here because the size here is based
@@ -159,6 +163,22 @@ bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) {
if (SamplePreInlineReplay)
return WasInlined;
+ // Check callee's inline attributes stamped from pseudo probe descriptors.
+ // always_inline functions should always be inlined; noinline functions
+ // should never be inlined.
+ if (Candidate.CalleeSamples->getContext().hasAttribute(ContextNoInline)) {
+ ++PreInlNumCSNotInlinedNoInline;
+ LLVM_DEBUG(dbgs() << " Noinline attribute for: "
+ << Candidate.CalleeSamples->getFunction() << "\n");
+ return false;
+ }
+ if (Candidate.CalleeSamples->getContext().hasAttribute(ContextAlwaysInline)) {
+ ++PreInlNumCSInlinedAlwaysInline;
+ LLVM_DEBUG(dbgs() << " AlwaysInline attribute for: "
+ << Candidate.CalleeSamples->getFunction() << "\n");
+ return true;
+ }
+
unsigned int SampleThreshold = SampleColdCallSiteThreshold;
uint64_t ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold(
(Summary->getDetailedSummary()));
@@ -225,7 +245,11 @@ void CSPreInliner::processFunction(const FunctionId Name) {
ContextTracker.markContextSamplesInlined(Candidate.CalleeSamples);
Candidate.CalleeSamples->getContext().setAttribute(
ContextShouldBeInlined);
- FuncFinalSize += Candidate.SizeCost;
+ // Don't charge the size budget for always_inline functions since the
+ // compiler will inline them regardless of the pre-inliner's decision.
+ if (!Candidate.CalleeSamples->getContext().hasAttribute(
+ ContextAlwaysInline))
+ FuncFinalSize += Candidate.SizeCost;
getInlineCandidates(CQueue, Candidate.CalleeSamples);
} else {
++PreInlNumCSNotInlined;
@@ -289,6 +313,26 @@ void CSPreInliner::run() {
LLVM_DEBUG(printProfileNames(ContextTracker, true));
+ // Stamp always_inline/noinline attributes from pseudo probe descriptors
+ // onto the corresponding FunctionSamples' contexts.
+ if (Binary.usePseudoProbes()) {
+ auto &FuncProfiles = ContextTracker.getFuncToCtxtProfiles();
+ for (const auto &FuncDesc : Binary.getGUID2FuncDescMap()) {
+ if (!FuncDesc.Attributes)
+ continue;
+ uint32_t CtxAttr = 0;
+ if (FuncDesc.Attributes &
+ (uint8_t)PseudoProbeDescAttributes::AlwaysInline)
+ CtxAttr |= ContextAlwaysInline;
+ if (FuncDesc.Attributes & (uint8_t)PseudoProbeDescAttributes::NoInline)
+ CtxAttr |= ContextNoInline;
+ FunctionId FuncId(FunctionSamples::getCanonicalFnName(FuncDesc.FuncName));
+ auto It = FuncProfiles.find(FuncId);
+ if (It != FuncProfiles.end())
+ for (auto *FSamples : It->second)
+ FSamples->getContext().setAttribute((ContextAttributeMask)CtxAttr);
+ }
+ }
// Execute global pre-inliner to estimate a global top-down inline
// decision and merge profiles accordingly. This helps with profile
// merge for ThinLTO otherwise we won't be able to merge profiles back
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index a437c57b8c0d7..c5f7bf7893586 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -668,6 +668,9 @@ class ProfiledBinary {
const MCPseudoProbeFuncDesc *getFuncDescForGUID(uint64_t GUID) {
return ProbeDecoder.getFuncDescForGUID(GUID);
}
+ const GUIDProbeFunctionMap &getGUID2FuncDescMap() const {
+ return ProbeDecoder.getGUID2FuncDescMap();
+ }
const MCPseudoProbeFuncDesc *
getInlinerDescForProbe(const MCDecodedPseudoProbe *Probe) {
More information about the llvm-commits
mailing list