[llvm] [llvm-profgen] Fix ContextWasInlined for intermediate inline frames (PR #191600)
Wei Wang via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 10 21:26:18 PDT 2026
https://github.com/apolloww created https://github.com/llvm/llvm-project/pull/191600
Add a `WasInlined` flag to `ContextTrieNode`, set from the probe inline tree during `getContextNodeForLeafProbe`. This stores the authoritative "was this function inlined in the binary" information on the trie node, making it available to any code that later creates `FunctionSamples` for the node.
`getOrCreateFunctionSamples` now checks both the caller-provided `WasLeafInlined` flag and the trie node's `WasInlined` flag, fixing the TODO where probeless functions would lose `ContextWasInlined` because their probes were never sampled directly.
>From 0c2c9134bc52342bf748cc74873014dff32d6ed5 Mon Sep 17 00:00:00 2001
From: Wei Wang <apollo.mobility at gmail.com>
Date: Fri, 10 Apr 2026 20:50:25 -0700
Subject: [PATCH] [CSProfileGenerator] Fix ContextWasInlined for intermediate
inline frames
Add a WasInlined flag to ContextTrieNode, set from the probe inline
tree during getContextNodeForLeafProbe. This stores the authoritative
"was this function inlined" information on the trie node, making it
available to any code that later creates FunctionSamples for the node.
getOrCreateFunctionSamples now checks both the caller-provided
WasLeafInlined flag and the trie node's WasInlined flag, fixing the
TODO where probeless functions would lose ContextWasInlined because
their probes were never sampled directly.
---
.../Transforms/IPO/SampleContextTracker.h | 6 ++++
.../Inputs/was-inlined-pseudoprobe.c | 26 +++++++++++++++
.../Inputs/was-inlined-pseudoprobe.perfbin | Bin 0 -> 9704 bytes
.../Inputs/was-inlined-pseudoprobe.perfscript | 5 +++
.../llvm-profgen/was-inlined-attribute.test | 24 ++++++++++++++
llvm/tools/llvm-profgen/ProfileGenerator.cpp | 31 +++++++++++-------
6 files changed, 81 insertions(+), 11 deletions(-)
create mode 100644 llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.c
create mode 100755 llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.perfbin
create mode 100644 llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.perfscript
create mode 100644 llvm/test/tools/llvm-profgen/was-inlined-attribute.test
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 1efe3ea127fd0..9f336c806dfec 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -59,6 +59,8 @@ class ContextTrieNode {
LLVM_ABI ContextTrieNode *getParentContext() const;
LLVM_ABI void setParentContext(ContextTrieNode *Parent);
LLVM_ABI void setCallSiteLoc(const LineLocation &Loc);
+ bool getWasInlined() const { return WasInlined; }
+ void setWasInlined(bool V) { WasInlined = V; }
LLVM_ABI void dumpNode();
LLVM_ABI void dumpTree();
@@ -80,6 +82,10 @@ class ContextTrieNode {
// Callsite location in parent context
LineLocation CallSiteLoc;
+
+ // Whether this function was inlined at this context in the binary.
+ // Set from the probe inline tree during profile generation.
+ bool WasInlined = false;
};
// Profile tracker that manages profiles and its associated context. It
diff --git a/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.c b/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.c
new file mode 100644
index 0000000000000..9d44bdd0e44d0
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.c
@@ -0,0 +1,26 @@
+#include <stdio.h>
+
+volatile int sink;
+
+__attribute__((always_inline))
+void work(int n) {
+ int s = 0;
+ for (int i = 1; i <= n; i++) {
+ if (i % 3)
+ s += i;
+ else
+ s -= i;
+ }
+ sink = s;
+}
+
+__attribute__((always_inline))
+void wrapper(int n) {
+ work(n);
+}
+
+int main() {
+ wrapper(4000 * 4000);
+ printf("result is %d\n", sink);
+ return 0;
+}
diff --git a/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.perfbin b/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.perfbin
new file mode 100755
index 0000000000000000000000000000000000000000..20f53a6b3696923561a1510a8657b9061cf96ceb
GIT binary patch
literal 9704
zcmd5i3v650b>}_uQ6E}A%d)K4ji1xVk%x*DWyzM6#E~q?rdByoWyPPa=?g_ZNn9vW
zARlAdXlLfrHcDgE?UrrWx-}WPA<H@>7`ng>vP`Z6!^^x>!5TDMhE7;q)Ipog&9*M{
z0CVTucMe58OIZ_O*sk#I`Ja2<x$l0dQ$r*Bbxi{)I_w3a7Mo1P>Qgnq^kzm7^g#~<
zvHm4!g8<Tg#dGC-U at Ao5wC?f+R=Onk5R%<9EPqBfl|<kcMF9wu2nmwiaxoco;iBP^
z0- at fe$&N(Am-iERKgS`k+>YL{g9GnqGm%1?;qx5WVufOZ*KVoG<uC0q6(X>W)jJr9
z9fd*d^>KTB+>XEkw<l2YIrbwbL3#N_<7785%63;dsaO3bdI1pFr6>S_lKW4g9mV-!
z3)$223g`P3?stJEI}|AGg;U9iU0vZ+qCJ&N=jYnzdUmz%>I&sDp-$zGx|!bTz}@#M
z5A=O;6N+A|BAfNA2pf at SuEM`%4gTIW_(zeyntzH;Q-40Jb+Ay6KUK)zl*F&b2TA6A
z`>Vf+sn#Knv=oY)@=x$4VG6V?%vRjWIkBu`&BT&vu&k+>Oqw|sSi|E7t%RMmr;<6x
z&W;}(NM+LYcx)nNyRua!v)N?YnFR7piQfeWMuztdSe>EHP<IL6>3%5-J4_Xg#3-R%
zEaR8=&te&$`cBH5WqkP-l&sfOfQB6pc|}t$%S+bI<x`HC%$co;Y7}UjeYaDDWV3_`
zA4OQ-fb$!|MTDP2*nu7Pb%bXSzJ!p<1{Z*KatyS&dTsN%ipn#XeB#p_^%(lvPCA1X
zsvD`IcAi2+;v>B%J|o!9XAnYvZT%PZftrAR1iiU_D2^vSK>irYFQGg^_WA%jxKG&=
z>tE|`z7~0Rc>eXzM(6)2`pD(0qvJ!7g~(q;Pv7}f47b?y*Z5`s+hnM2>!YNu^v_>f
zE~xkV$1%*)#;XMOyzewczij*w8b!ab;8Yct>b4fh?mR=(e_H8x{|<p2*Q4`Sqv!tl
zuIRb<eNpW~^!4k`26QmP4l0XFlXY7Mm+dLu!kwQ-TgW%w8=e2rm6OF{@q+OHa<u!-
zpE*Cd9BbXyJ{9jz9$$`^Od<<kDE`_ne}{&`%V=DD at lT$8_F0avfBxb~-E-d^K6ho?
zfz$rJe(h?1 at nZiY?-x-pST}fKP+KUT8$9jbjqKuUB*<TyH{QcxejmPKAN=q5|60j^
zWI at IKBE|hO#XYG?sek6b7p;5ldi31CY&(6&DeVXI->VzEroCT0SA6xMYsI&M#rzK`
z!g-?)J$|{n7i;X1ig$~}$ikIAOh{zmzVnph=p#R^#C~`KKOO2wy|@^C<dw?vDxXTG
zDEvUW|9%_Rt^a^Ey??QlX!<JQoh6L1u+bzOo at Yo&r$X}=l_nRIYFQSK^16bbxhnTF
z<mC}IG2P7YEbHq`i`>Wb6<$weTuM7M-%z=tn+gV14JH3F7QgRv<uoTyDX1FY->(-l
zgq~%23-fz;9hPm7yob!avLB!^PDSMFToNufn+mI88aBD~)uP+p9JhC!=|`Eyl&g3o
z&tm>zq3j>LcH23Qcq*2jG9R?Fxnw47M)rgvp-!`H+Hq!cz2Wdw(h1?*Z)el7RLGf}
zNFNWSZ6}<yXEQkxhO@`RncR%+#KQKRs}@eB9-L{P&1QbpjyvW=S4Veed?FsP6CIQB
z$vufkPv>O9-aQfN>FKZ&U3(H;-QBx8CU at ZMm`p!lXM4?&k-`7VI1UU9^qOtBo!DVU
zBBAb3huPWD+10T-vd3&2vlC`C=CGhW(zOEy_V at Ofp-?z?BA2jdb79<{gkuxQ_H at Rv
zLva{8+?&m((@D(nR6OqW2>VBR+an)ferTXK0tfoBN*?O%a2vzb-ATsVvvw*I#}*C^
zq2R!g-j2DQiO!DhT`@D6#$Jm<|KZ-Q*+}P}o|RIMxr(1H4vqHigwbKF_8snxrPG;-
zWcmZbW~&u#ubm0^RXwe{<3%l;_ at N)@zg62Ai<HhFKW}o!3XLhRj=}Sgz}0kQ>|NtA
zxSEcP$4VX#tLfBpTQ4wNO-I at i_)&Ev*5U`mEV0&b%v0}t;!E})zjUfw+xcD_rpkXj
zlYQWGw;i$`d-0u5(E@`@YJJ7=Y;1Pc&R%=t^pAHXo_x#Y>)@w-;xuO1z`2WNQJQgE
z@#(~;AD>Ztj^g9 at xA+?18~*UPHtG_ZBF4a<IH4q8(Y~#bj7XFz5~c5IzpE4bUF~6A
z!(CK^*4o-=_*%2g_05}G8#b>G<eHz*Z~@S}V7Re2czaOC7<`&GjGG7lHstNb+1$7l
ze64$fK|}+D27`MMt>{8 at xbdzQF{|uGeSoTvMvnV7ootC9S#5y6iI89y=(iBEE~pd$
zYC at W?m8@@TxvRz8hM!@rfj)^tj_RlZA?kxDZupoMX(Z=IgU5nLf-mgDq^)0WPPFf8
z0cbQ}OAWQQAquA&KaXoj6SrpE)-v392et^X(NGaLQ{Ff2R(MOD0yk|}0H&~212p=i
zaNqwi#*onUTTw)OT5j|RR<KEb8hm#m;_t_*77SlN3-~K5Dh%pK at aa!$m2^}x0*1dz
zr|qbAxPW!Fb}#J=gYX2_>wI?;u04r0rl<k}wfJ$6SzQ%W5Uj7O2IU8H?RUNWBeK!U
z+Xxy8SQ+D3L8Gr3%+2CmSKbZf-RF9Tnjox4_v^_mtPdox)(+ReMxq<YX~ie7f at bBY
zv2by*qSS;dahp&Q^$LE~NX0E^6g0Sz4WCq*YFej#RwKt7e+B7g*mOUF78|RZC}(g7
zKD1Tv<MVZVUdQK8 at p%EC-@@n5 at VSnUjt_f=(}=%{&olTWAr1d;mQ)_`kb at kqtq^u*
zX2Xu1qs2iT4&&3YEa38vr^NoTvHoM$q2W&tS>wk>hb#;D&=*d`aB<J)aDxy&j$4J}
zb}~FWl?f-)cpkUi#h#`cCu`e42#jDFXf9deo!mM;u`TfId4ZwHGu(ED1%^*CTw+M+
zCV%w#YMS<6wKW8=$LI`<$W|Q(kt#mK>x`Rqwb#0I`^jTVKYs6_z+c?__<!D3|C9Y>
z at c4T*Z~uAKYj4i&dUD{KO--ZN%~iSt)@(FYPvdY2`Q}1KRlrwOdDI97E*n+9Z)^$t
zp3xKt8NQ1~WuVK at f>Q=0QQi08<S%HDGGN-Ms`lk7FQe0oMpX!%=3J)_qtleJ#aE5a
zAmIN-q40NFU^4K7Krp1^ZGndA&tk&kseHm-(Tg9db8=21nF&pUEC6`>R{-C`Z-FjL
z1!%e~C6%BR{_vDu0Dr-(C_rTa5cd_fR~29lZNF09Ql{L6%H{r6*KoUT&D!l}9q)}y
zrQ_BBG%x7X)BH8jpru0len0#Sm%No_Ba#{pcsW0a^y7%=Py at 2Y3%`f?ee7TOpJaZ)
z!;c|fgBGadubtwiGJ}r>4W)ZS#l-42 at S%3)JSF_65Yh1FNX{L?e~vzOy&B%+Q6>Dp
zL}a!1y|4zKCNvF~i#EeVY`M%R|2O8RmR<zcV{hKxQ2a*lo-bm^*Y(=cxkp-{HB8s*
zaD6g`rQm;Hy&m%*_p#eqN~50mGAksQ??XQ(Yy`PaWj4 at 4*Z|eA$oZiOoXSq*<8{sr
zWeEB8tN0m1zIlT-z=<{ZPp-j#3HcQN0)Hp8L{d42{096^WLA_oFR;GJv!3X`rueSy
zirPJw%|b|>H2U`qw>z;Z2qjLWb0=mXG?mVW_*A!ou(B>W9m`F-+75E?FcBkD#%5Cv
zVi}tHA!JWmliApeZA~XoQz!^Ic;W+$B8l}R0thKHLakUf8$00&1xN3(nPePW&yaIF
zmCabQu|xuoMi_@`$(ga at bRTIs>Ozt<U7r)V9E9STnHd`ws}P?|)QcmZ-gF|LOeI1w
z$H^uq^0>rdEIB)$z!tKZ2^()bbJ%dgp2$yGcxp=F<tM7RbZlZGYd<JdGCi3QJm+Ms
z%;aRwURFkyNHO#hPD`Qi3CH_Zl$D17A?y`=Erio3bTJp=>Wc?PeCH{MXPumr$D6M>
zGVyZO8W}z`j%)Gqjjbg)%7I+fma!}gUwJB(nTVyVgp<kUtXO^yFmbaf+p!a&4s^SU
zhPB*h7G^km0^ANy<Y#71ph+2JQGdH#%BsAfSP}aEvHpWY*3jL97`v57S?a7Bb0{x#
zcC}`cc- at S6)^Rr?)cs6b2S)DM*FR$2vw#1gp>b=xf8WRu-CN&-a{N{-+=(PGq#N(K
z%YAq*z126WUXs(wk$t4gaLiyXV@=1>3A{5`8y>Z}lD!sho2<cOclRG09#}DCKMH}v
zALnaYE8HT#lgaOATbM6^bju6fv(X)y*vmR9zr&d2%81K+B<yBHXiXJ+xnH}&j5DlA
zZBX0lkb;m-GgP+XLw8wXFZW$@J|$KM7s8k<wIlkp^{2Zrq2+$;Q8wss-Kck at _Pz1n
zj=Z0j*~|UjS+<w_N&6B%yM-Y{NG9#e{o(i7pdgA2z4<?ge2Piz<v#Q%2YNr-dE1x#
z9bx- at tS9%QXW3r#$ewVCU-&WPQA~82ry}d)BHQ2WvP-qt3nPgL*-QNL`=I=8*d>Y#
z*=>n?9I<kH*>|+ at hV7Y>ri*yne~9e^tS9?~MYi7{fw8=vp at 1(TLhXya+|OL*i(@>K
zuPEO5PouQlzO;YgKye4UqIm6p-D6+R`=5HYH&<BmJy#j;w-Lu(Q)zt2eY5=D>&=(Y
z0>6p2wPp6Quk2w3(L)tggckTbaw&K6eae2Xj};1(94ws9()TC*@Ew$qirCBVg9|h<
zVKG&`TuHmac^MJniT%<JQ;FpFbyGT4aox5URp4*1E^lA<jbiWZFMgepi@){Q%RcxD
z$6r!p#2f$H|AW2hap1N8Day;=U(Oem`~!wp-}^p4<Mw5ql;*^~&-21U3cR+4eUugj
zEPF-KB~`8o6?PRO@@=q+0*|e8sX<X>p*Q}mDEFu+%}YG+WL{+R8+1F at u&<>-fhFaF
zQtfSP-x~HO*`EBD7Y_-eYuJB_1~rxkJO&;TKC^~>bAxN&=Ymr0wa>7<xBri at J)Nh@
zi;x0OvVFM@;h*M1)u=}xcEbMR8urh!z3wsaknlCO_s0J%yj7(_H;Ls%>;?V-+ZW1}
z5WmLuUtzw?<6<Z3^nX5{B}R$PY;ZL?cw*XtWD`Cz9%cPKi}o^ixfX7{KpQJ8uX-Bv
Tknr6#>>uY#uOE60JS6-#zaEW;
literal 0
HcmV?d00001
diff --git a/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.perfscript b/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.perfscript
new file mode 100644
index 0000000000000..76a5d1e5d9bd8
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/Inputs/was-inlined-pseudoprobe.perfscript
@@ -0,0 +1,5 @@
+PERF_RECORD_MMAP2 595196/595196: [0x201000(0x1000) @ 0 00:1d 224227621 1042948]: r-xp /home/was-inlined-pseudoprobe.perfbin
+
+ 2017a0
+ 5541f689495641d7
+ 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x2017c0/0x2017a0/P/-/-/0 0x0/0x201780/P/-/-/0
diff --git a/llvm/test/tools/llvm-profgen/was-inlined-attribute.test b/llvm/test/tools/llvm-profgen/was-inlined-attribute.test
new file mode 100644
index 0000000000000..cfa3895651f2b
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/was-inlined-attribute.test
@@ -0,0 +1,24 @@
+; Test that ContextWasInlined (!Attributes: 1) is correctly set for probeless
+; inlined functions via the WasInlined flag on ContextTrieNode.
+;
+; The test binary has main → wrapper → work inlined. wrapper's probes were
+; dropped (compiled with -drop-pseudo-probes-for=wrapper), making it probeless.
+; The WasInlined flag is set on wrapper's trie node from the probe inline tree
+; during probe processing. wrapper gets FunctionSamples via the existing
+; "infer caller from callee entry probe" path, and ContextWasInlined is set
+; from the trie node's WasInlined flag in getOrCreateFunctionSamples.
+;
+; Without the WasInlined fix, wrapper would NOT have !Attributes: 1.
+
+; RUN: llvm-profgen --format=text --perfscript=%S/Inputs/was-inlined-pseudoprobe.perfscript --binary=%S/Inputs/was-inlined-pseudoprobe.perfbin --output=%t --profile-summary-cold-count=0 --csspgo-preinliner=0 --gen-cs-nested-profile=0
+; RUN: FileCheck %s --input-file %t
+
+; work is inlined and sampled — should have Attributes: 1.
+; CHECK: [main:2 @ wrapper:2 @ work]:
+; CHECK: !Attributes: 1
+; main is NOT inlined — should NOT have Attributes.
+; CHECK: [main]:
+; CHECK-NOT: !Attributes:
+; wrapper is inlined but probeless — should have Attributes: 1
+; CHECK: [main:2 @ wrapper]:
+; CHECK: !Attributes: 1
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index c3f489040007e..9de710b420630 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -896,17 +896,10 @@ CSProfileGenerator::getOrCreateFunctionSamples(ContextTrieNode *ContextNode,
FProfile->setFunction(ContextNode->getFuncName());
ContextNode->setFunctionSamples(FProfile);
}
- // Update ContextWasInlined attribute for existing contexts.
- // The current function can be called in two ways:
- // - when processing a probe of the current frame
- // - when processing the entry probe of an inlinee's frame, which
- // is then used to update the callsite count of the current frame.
- // The two can happen in any order, hence here we are making sure
- // `ContextWasInlined` is always set as expected.
- // TODO: Note that the former does not always happen if no probes of the
- // current frame has samples, and if the latter happens, we could lose the
- // attribute. This should be fixed.
- if (WasLeafInlined)
+ // Update ContextWasInlined attribute. Use either the caller-provided flag
+ // or the WasInlined flag stored on the trie node (set during probe
+ // processing from the probe inline tree).
+ if (WasLeafInlined || ContextNode->getWasInlined())
FProfile->getContext().setAttribute(ContextWasInlined);
return FProfile;
}
@@ -1412,6 +1405,22 @@ ContextTrieNode *CSProfileGenerator::getContextNodeForLeafProbe(
ContextTrieNode *ContextNode =
getOrCreateContextNode(NewContextStack, WasLeafInlined);
ContextNode->getFunctionSamples()->setFunctionHash(FuncDesc->FuncHash);
+
+ // Mark all intermediate inline frames as inlined on the trie node. Walk up
+ // the probe inline tree (which tells us which functions were inlined) and
+ // set the flag on corresponding trie nodes. This information is used later
+ // when creating FunctionSamples (e.g. for probeless nodes) to set
+ // ContextWasInlined.
+ ContextTrieNode *CurNode = ContextNode;
+ auto *InlineTreeNode = LeafProbe->getInlineTreeNode();
+ while (InlineTreeNode->hasInlineSite() && CurNode &&
+ CurNode != &ContextTracker.getRootContext()) {
+ CurNode->setWasInlined(true);
+ InlineTreeNode =
+ static_cast<MCDecodedPseudoProbeInlineTree *>(InlineTreeNode->Parent);
+ CurNode = CurNode->getParentContext();
+ }
+
return ContextNode;
}
More information about the llvm-commits
mailing list