[llvm-branch-commits] [llvm] [ProfileData] Handle .cfi suffixes in InstrProf and Coro SampleProf (PR #240698)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 9 20:17:36 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-pgo
Author: Vitaly Buka (vitalybuka)
<details>
<summary>Changes</summary>
Under CFI jump tables (`-fsanitize=cfi-icall`), `LowerTypeTests` renames
function definitions to `<name>.cfi` while `<name>` becomes a jump-table
declaration or naked stub. `InstrProfSymtab` lacked `.cfi` suffix
stripping and could map a canonical GUID to a jump-table declaration or
naked stub instead of the real `.cfi` body when the stub appeared
earlier in the module. Similarly, `FunctionSamples::getCanonicalCoroFnName`
did not strip `.cfi` when canonicalizing coroutine clones for pseudo
probes and `llvm-profgen`.
CC: @<!-- -->benschlueter
Co-authored-by: Benedict Schlüter <bschlueter@<!-- -->google.com>
Assisted-by: Gemini
---
Full diff: https://github.com/llvm/llvm-project/pull/240698.diff
5 Files Affected:
- (modified) llvm/include/llvm/ProfileData/InstrProf.h (+2-2)
- (modified) llvm/include/llvm/ProfileData/SampleProf.h (+5-4)
- (modified) llvm/lib/ProfileData/InstrProf.cpp (+28-15)
- (modified) llvm/unittests/ProfileData/InstrProfTest.cpp (+68-1)
- (modified) llvm/unittests/ProfileData/SampleProfTest.cpp (+11)
``````````diff
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index c0075ea6390f4..1fccebee7320f 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -502,8 +502,8 @@ class InstrProfSymtab {
using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
// Returns the canonical name of the given PGOName. This shares the same
- // logic of FunctionSamples::getCanonicalFnName() but only strips ".llvm."
- // and ".part.", and leaves out ".__uniq.".
+ // logic of FunctionSamples::getCanonicalFnName() but only strips ".cfi",
+ // ".llvm.", and ".part.", and leaves out ".__uniq.".
LLVM_ABI static StringRef getCanonicalName(StringRef PGOName);
private:
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index b2d42ed61c473..72b54eb876993 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -1333,10 +1333,11 @@ class FunctionSamples {
StringRef Attr = "selected") {
// A local coroutine function from another CU can be promoted to a global
// function during ThinLTO import. This will create a linkage name like
- // "_Zfoo.llvm.xxxx.cleanup". Remove the ".llvm." suffix after stripping all
- // the coroutine suffixes to avoid pseudo probe mismatch.
- const SmallVector<StringRef, 4> CoroSuffixes{".cleanup", ".destroy",
- ".resume", LLVMSuffix};
+ // "_Zfoo.llvm.xxxx.cleanup". Under CFI, LowerTypeTests may also append
+ // ".cfi" either before or after CoroSplit appends the coroutine suffix.
+ // Remove these suffixes to avoid pseudo probe mismatch.
+ const SmallVector<StringRef, 6> CoroSuffixes{
+ CfiSuffix, ".cleanup", ".destroy", ".resume", CfiSuffix, LLVMSuffix};
return getCanonicalFnName(FnName, CoroSuffixes, Attr);
}
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index e8c2a5179ae5d..b70c94b4a19dd 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -483,21 +483,33 @@ GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
Error InstrProfSymtab::create(Module &M, bool InLTO, bool AddCanonical,
bool KeyImportedAliasesByAliasGUID) {
- for (Function &F : M) {
- // Function may not have a name: like using asm("") to overwrite the name.
- // Ignore in this case.
- if (!F.hasName())
- continue;
- auto IRPGOFuncName = getIRPGOObjectName(F, InLTO);
- if (Error E = addFuncWithName(F, IRPGOFuncName, AddCanonical,
- KeyImportedAliasesByAliasGUID))
- return E;
- // Also use getPGOFuncName() so that we can find records from older profiles
- auto PGOFuncName = getPGOFuncName(F, InLTO);
- if (PGOFuncName != IRPGOFuncName)
- if (Error E = addFuncWithName(F, PGOFuncName, AddCanonical,
+ auto IsPreferredDef = [](const Function &F) {
+ return !F.isDeclaration() && !F.hasFnAttribute(Attribute::Naked);
+ };
+ // Process non-naked definitions before declarations and naked jump-table
+ // stubs so that when both share a canonical name or GUID (for example, a CFI
+ // function definition @foo.cfi and its jump-table declaration or stub @foo),
+ // finalizeSymtab()'s stable_sort keeps the real definition first.
+ for (bool VisitPreferred : {true, false}) {
+ for (Function &F : M) {
+ if (IsPreferredDef(F) != VisitPreferred)
+ continue;
+ // Function may not have a name: like using asm("") to overwrite the name.
+ // Ignore in this case.
+ if (!F.hasName())
+ continue;
+ auto IRPGOFuncName = getIRPGOObjectName(F, InLTO);
+ if (Error E = addFuncWithName(F, IRPGOFuncName, AddCanonical,
KeyImportedAliasesByAliasGUID))
return E;
+ // Also use getPGOFuncName() so that we can find records from older
+ // profiles
+ auto PGOFuncName = getPGOFuncName(F, InLTO);
+ if (PGOFuncName != IRPGOFuncName)
+ if (Error E = addFuncWithName(F, PGOFuncName, AddCanonical,
+ KeyImportedAliasesByAliasGUID))
+ return E;
+ }
}
for (GlobalVariable &G : M.globals()) {
@@ -616,7 +628,7 @@ StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) {
//
// ".__uniq." suffix is used to differentiate internal linkage functions in
// different modules and should be kept. This is the only suffix with the
- // pattern ".xxx" which is kept before matching, other suffixes ".llvm." and
+ // pattern ".xxx" which is kept before matching; ".cfi", ".llvm.", and
// ".part." will be stripped.
//
// Leverage the common canonicalization logic from FunctionSamples. Instead of
@@ -628,7 +640,8 @@ StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) {
// and its post-split clones (foo.resume, foo.cleanup) are all canonicalized
// to "foo" otherwise, which can make the symtab lookup return unexpected
// result.
- const SmallVector<StringRef> SuffixesToRemove{FunctionSamples::LLVMSuffix,
+ const SmallVector<StringRef> SuffixesToRemove{FunctionSamples::CfiSuffix,
+ FunctionSamples::LLVMSuffix,
FunctionSamples::PartSuffix};
return FunctionSamples::getCanonicalFnName(PGOName, SuffixesToRemove);
}
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index a7fe530c60764..e70f268efa803 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -814,12 +814,79 @@ TEST_F(InstrProfTest, test_symtab_lookup_vtable_with_assigned_guid) {
TEST_F(InstrProfTest, test_symtab_get_canonical_name) {
EXPECT_EQ(InstrProfSymtab::getCanonicalName("foo.llvm.1234"), "foo");
EXPECT_EQ(InstrProfSymtab::getCanonicalName("foo.part.1"), "foo");
+ EXPECT_EQ(InstrProfSymtab::getCanonicalName("foo.cfi"), "foo");
EXPECT_EQ(
InstrProfSymtab::getCanonicalName("foo.__uniq.123.part.1.llvm.9999"),
"foo.__uniq.123");
- // Coroutine clone suffixes are preserved.
+ EXPECT_EQ(InstrProfSymtab::getCanonicalName(
+ "foo.__uniq.123.llvm.13757e0fb71915e385efa4dc9d1e08fd.cfi"),
+ "foo.__uniq.123");
+ EXPECT_EQ(
+ InstrProfSymtab::getCanonicalName("foo.__uniq.123.part.1.llvm.9999.cfi"),
+ "foo.__uniq.123");
+ // Coroutine clone and non-canonical jump-table suffixes are preserved.
EXPECT_EQ(InstrProfSymtab::getCanonicalName("coro.llvm.1234.resume"),
"coro.llvm.1234.resume");
+ EXPECT_EQ(InstrProfSymtab::getCanonicalName("foo.cfi_jt"), "foo.cfi_jt");
+}
+
+// Verify that non-naked definitions take precedence over CFI jump-table
+// declarations and naked jump-table stubs in InstrProfSymtab regardless of
+// symbol order in the module, while standalone declarations remain reachable.
+TEST_F(InstrProfTest, test_symtab_cfi_def_precedence) {
+ LLVMContext Ctx;
+ auto M = std::make_unique<Module>("MyModule.cpp", Ctx);
+ auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false);
+
+ // Place jump-table declaration BEFORE the .cfi definition in module order.
+ auto *JTDecl = Function::Create(
+ FTy, Function::ExternalLinkage,
+ "InternalLambda.__uniq.42.llvm.13757e0fb71915e385efa4dc9d1e08fd",
+ M.get());
+ (void)JTDecl;
+
+ auto *CfiDef = Function::Create(
+ FTy, Function::ExternalLinkage,
+ "InternalLambda.__uniq.42.llvm.13757e0fb71915e385efa4dc9d1e08fd.cfi",
+ M.get());
+ IRBuilder<> Builder(BasicBlock::Create(Ctx, "entry", CfiDef));
+ Builder.CreateRetVoid();
+
+ // Simulate PreLink !guid metadata assigned while linkage was internal.
+ const uint64_t PreLinkGUID = Function::getGUIDAssumingExternalLinkage(
+ "MyModule.cpp;InternalLambda.__uniq.42");
+ CfiDef->setMetadata(
+ LLVMContext::MD_guid,
+ MDNode::get(Ctx, {ConstantAsMetadata::get(ConstantInt::get(
+ Type::getInt64Ty(Ctx), PreLinkGUID))}));
+
+ // Place a naked jump-table stub BEFORE its .cfi definition in module order.
+ auto *NakedStub =
+ Function::Create(FTy, Function::ExternalLinkage, "NakedStub", M.get());
+ NakedStub->addFnAttr(Attribute::Naked);
+ IRBuilder<>(BasicBlock::Create(Ctx, "entry", NakedStub)).CreateUnreachable();
+
+ auto *NakedCfiDef = Function::Create(FTy, Function::ExternalLinkage,
+ "NakedStub.cfi", M.get());
+ IRBuilder<>(BasicBlock::Create(Ctx, "entry", NakedCfiDef)).CreateRetVoid();
+
+ // Standalone declaration without a local definition must still be found.
+ auto *ExtOnly =
+ Function::Create(FTy, Function::ExternalLinkage, "ExtOnly", M.get());
+
+ InstrProfSymtab Symtab;
+ EXPECT_THAT_ERROR(Symtab.create(*M, /*InLTO=*/true), Succeeded());
+
+ const uint64_t CanonicalGUID =
+ Function::getGUIDAssumingExternalLinkage("InternalLambda.__uniq.42");
+ EXPECT_EQ(Symtab.getFunction(CanonicalGUID), CfiDef);
+ EXPECT_EQ(Symtab.getFunction(PreLinkGUID), CfiDef);
+ EXPECT_EQ(
+ Symtab.getFunction(Function::getGUIDAssumingExternalLinkage("NakedStub")),
+ NakedCfiDef);
+ EXPECT_EQ(
+ Symtab.getFunction(Function::getGUIDAssumingExternalLinkage("ExtOnly")),
+ ExtOnly);
}
// callee1 to callee6 are from vtable1 to vtable6 respectively.
diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp
index 6dd7b701dec5f..bb0f9468bbeea 100644
--- a/llvm/unittests/ProfileData/SampleProfTest.cpp
+++ b/llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -1331,7 +1331,18 @@ TEST(SampleProfCanonicalNameTest, CoroFnName) {
FunctionSamples::getCanonicalCoroFnName("_Zfoo.llvm.1234.destroy"));
EXPECT_EQ("_Zfoo",
FunctionSamples::getCanonicalCoroFnName("_Zfoo.llvm.1234.resume"));
+ EXPECT_EQ("foo", FunctionSamples::getCanonicalCoroFnName("foo.cfi"));
+ EXPECT_EQ("_Zfoo", FunctionSamples::getCanonicalCoroFnName(
+ "_Zfoo.llvm.1234.resume.cfi"));
+ EXPECT_EQ("_Zfoo", FunctionSamples::getCanonicalCoroFnName(
+ "_Zfoo.llvm.1234.cfi.resume"));
+ EXPECT_EQ("_Zfoo", FunctionSamples::getCanonicalCoroFnName(
+ "_Zfoo.llvm.1234.cleanup.cfi"));
+ EXPECT_EQ("_Zfoo", FunctionSamples::getCanonicalCoroFnName(
+ "_Zfoo.llvm.1234.cfi.destroy"));
// Suffixes without trailing dot require an exact match.
+ EXPECT_EQ("foo.cfi_jt",
+ FunctionSamples::getCanonicalCoroFnName("foo.cfi_jt"));
EXPECT_EQ("foo.cleanupper",
FunctionSamples::getCanonicalCoroFnName("foo.cleanupper"));
EXPECT_EQ("foo.destroyer",
``````````
</details>
https://github.com/llvm/llvm-project/pull/240698
More information about the llvm-branch-commits
mailing list