[llvm] 1e15371 - [ThinLTO][TypeProf] Implement vtable def import (#79381)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 1 15:14:52 PDT 2024
Author: Mingming Liu
Date: 2024-04-01T15:14:49-07:00
New Revision: 1e15371dd8843dfc52b9435afaa133997c1773d8
URL: https://github.com/llvm/llvm-project/commit/1e15371dd8843dfc52b9435afaa133997c1773d8
DIFF: https://github.com/llvm/llvm-project/commit/1e15371dd8843dfc52b9435afaa133997c1773d8.diff
LOG: [ThinLTO][TypeProf] Implement vtable def import (#79381)
Add annotated vtable GUID as referenced variables in per function
summary, and update bitcode writer to create value-ids for these
referenced vtables.
- This is the part3 of type profiling work, and described in the "Virtual Table Definition Import" [1] section of the
RFC.
[1] https://github.com/llvm/llvm-project/pull/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
Added:
Modified:
llvm/include/llvm/ProfileData/InstrProf.h
llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/ProfileData/InstrProf.cpp
llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index fd66c4ed948f36..eb3c10bcba1ca7 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -283,7 +283,7 @@ void annotateValueSite(Module &M, Instruction &Inst,
/// Extract the value profile data from \p Inst which is annotated with
/// value profile meta data. Return false if there is no value data annotated,
-/// otherwise return true.
+/// otherwise return true.
bool getValueProfDataFromInst(const Instruction &Inst,
InstrProfValueKind ValueKind,
uint32_t MaxNumValueData,
@@ -291,6 +291,16 @@ bool getValueProfDataFromInst(const Instruction &Inst,
uint32_t &ActualNumValueData, uint64_t &TotalC,
bool GetNoICPValue = false);
+/// Extract the value profile data from \p Inst and returns them if \p Inst is
+/// annotated with value profile data. Returns nullptr otherwise. It's similar
+/// to `getValueProfDataFromInst` above except that an array is allocated only
+/// after a preliminary checking that the value profiles of kind `ValueKind`
+/// exist.
+std::unique_ptr<InstrProfValueData[]>
+getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
+ uint32_t MaxNumValueData, uint32_t &ActualNumValueData,
+ uint64_t &TotalC, bool GetNoICPValue = false);
+
inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
/// Return the PGOFuncName meta data associated with a function.
diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
index ebfa1c8fc08e1c..ab53717eb889a0 100644
--- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
+++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
@@ -45,6 +45,10 @@ static cl::opt<unsigned>
cl::desc("Max number of promotions for a single indirect "
"call callsite"));
+cl::opt<unsigned> MaxNumVTableAnnotations(
+ "icp-max-num-vtables", cl::init(6), cl::Hidden,
+ cl::desc("Max number of vtables annotated for a vtable load instruction."));
+
ICallPromotionAnalysis::ICallPromotionAnalysis() {
ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumPromotions);
}
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 1f15e94783240a..3ad0bab827a512 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -82,6 +82,8 @@ static cl::opt<std::string> ModuleSummaryDotFile(
extern cl::opt<bool> ScalePartialSampleProfileWorkingSetSize;
+extern cl::opt<unsigned> MaxNumVTableAnnotations;
+
// Walk through the operands of a given User via worklist iteration and populate
// the set of GlobalValue references encountered. Invoked either on an
// Instruction or a GlobalVariable (which walks its initializer).
@@ -124,6 +126,24 @@ static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
Worklist.push_back(Operand);
}
}
+
+ const Instruction *I = dyn_cast<Instruction>(CurUser);
+ if (I) {
+ uint32_t ActualNumValueData = 0;
+ uint64_t TotalCount = 0;
+ // MaxNumVTableAnnotations is the maximum number of vtables annotated on
+ // the instruction.
+ auto ValueDataArray =
+ getValueProfDataFromInst(*I, IPVK_VTableTarget, MaxNumVTableAnnotations,
+ ActualNumValueData, TotalCount);
+
+ if (ValueDataArray.get()) {
+ for (uint32_t j = 0; j < ActualNumValueData; j++) {
+ RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */
+ ValueDataArray[j].Value));
+ }
+ }
+ }
return HasBlockAddress;
}
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 221eeaae6e2b83..dd554e422516f6 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -203,7 +203,7 @@ class ModuleBitcodeWriterBase : public BitcodeWriterBase {
for (const auto &GUIDSummaryLists : *Index)
// Examine all summaries for this GUID.
for (auto &Summary : GUIDSummaryLists.second.SummaryList)
- if (auto FS = dyn_cast<FunctionSummary>(Summary.get()))
+ if (auto FS = dyn_cast<FunctionSummary>(Summary.get())) {
// For each call in the function summary, see if the call
// is to a GUID (which means it is for an indirect call,
// otherwise we would have a Value for it). If so, synthesize
@@ -211,6 +211,15 @@ class ModuleBitcodeWriterBase : public BitcodeWriterBase {
for (auto &CallEdge : FS->calls())
if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
assignValueId(CallEdge.first.getGUID());
+
+ // For each referenced variables in the function summary, see if the
+ // variable is represented by a GUID (as opposed to a symbol to
+ // declarations or definitions in the module). If so, synthesize a
+ // value id.
+ for (auto &RefEdge : FS->refs())
+ if (!RefEdge.haveGVs() || !RefEdge.getValue())
+ assignValueId(RefEdge.getGUID());
+ }
}
protected:
@@ -4188,7 +4197,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
NameVals.push_back(SpecialRefCnts.second); // worefcnt
for (auto &RI : FS->refs())
- NameVals.push_back(VE.getValueID(RI.getValue()));
+ NameVals.push_back(getValueId(RI));
const bool UseRelBFRecord =
WriteRelBFToSummary && !F.hasProfileData() &&
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 90c3cfc45b98ae..95f900d0fff1ca 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -1271,46 +1271,44 @@ void annotateValueSite(Module &M, Instruction &Inst,
Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
}
-bool getValueProfDataFromInst(const Instruction &Inst,
- InstrProfValueKind ValueKind,
- uint32_t MaxNumValueData,
- InstrProfValueData ValueData[],
- uint32_t &ActualNumValueData, uint64_t &TotalC,
- bool GetNoICPValue) {
+MDNode *mayHaveValueProfileOfKind(const Instruction &Inst,
+ InstrProfValueKind ValueKind) {
MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
if (!MD)
- return false;
+ return nullptr;
- unsigned NOps = MD->getNumOperands();
+ if (MD->getNumOperands() < 5)
+ return nullptr;
- if (NOps < 5)
- return false;
-
- // Operand 0 is a string tag "VP":
MDString *Tag = cast<MDString>(MD->getOperand(0));
- if (!Tag)
- return false;
-
- if (!Tag->getString().equals("VP"))
- return false;
+ if (!Tag || !Tag->getString().equals("VP"))
+ return nullptr;
// Now check kind:
ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
if (!KindInt)
- return false;
+ return nullptr;
if (KindInt->getZExtValue() != ValueKind)
- return false;
+ return nullptr;
+
+ return MD;
+}
+static bool getValueProfDataFromInstImpl(const MDNode *const MD,
+ const uint32_t MaxNumDataWant,
+ InstrProfValueData ValueData[],
+ uint32_t &ActualNumValueData,
+ uint64_t &TotalC, bool GetNoICPValue) {
+ const unsigned NOps = MD->getNumOperands();
// Get total count
ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
if (!TotalCInt)
return false;
TotalC = TotalCInt->getZExtValue();
-
ActualNumValueData = 0;
for (unsigned I = 3; I < NOps; I += 2) {
- if (ActualNumValueData >= MaxNumValueData)
+ if (ActualNumValueData >= MaxNumDataWant)
break;
ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
ConstantInt *Count =
@@ -1327,6 +1325,36 @@ bool getValueProfDataFromInst(const Instruction &Inst,
return true;
}
+std::unique_ptr<InstrProfValueData[]>
+getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
+ uint32_t MaxNumValueData, uint32_t &ActualNumValueData,
+ uint64_t &TotalC, bool GetNoICPValue) {
+ MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind);
+ if (!MD)
+ return nullptr;
+ auto ValueDataArray = std::make_unique<InstrProfValueData[]>(MaxNumValueData);
+ if (!getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueDataArray.get(),
+ ActualNumValueData, TotalC, GetNoICPValue))
+ return nullptr;
+ return ValueDataArray;
+}
+
+// FIXME: Migrate existing callers to the function above that returns an
+// array.
+bool getValueProfDataFromInst(const Instruction &Inst,
+ InstrProfValueKind ValueKind,
+ uint32_t MaxNumValueData,
+ InstrProfValueData ValueData[],
+ uint32_t &ActualNumValueData, uint64_t &TotalC,
+ bool GetNoICPValue) {
+ MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind);
+ if (!MD)
+ return false;
+ return getValueProfDataFromInstImpl(MD, MaxNumValueData, ValueData,
+ ActualNumValueData, TotalC,
+ GetNoICPValue);
+}
+
MDNode *getPGOFuncNameMetadata(const Function &F) {
return F.getMetadata(getPGOFuncNameMetadataName());
}
diff --git a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll
index 78b175caca85f0..ba3ce9a75ee832 100644
--- a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll
+++ b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll
@@ -1,20 +1,31 @@
-; RUN: opt -module-summary %s -o %t.o
+; Promote at most one function and annotate at most one vtable.
+; As a result, only one value (of each relevant kind) shows up in the function
+; summary.
+
+; RUN: opt -module-summary -icp-max-num-vtables=1 -icp-max-prom=1 %s -o %t.o
; RUN: llvm-bcanalyzer -dump %t.o | FileCheck %s
; RUN: llvm-dis -o - %t.o | FileCheck %s --check-prefix=DIS
-
+; Round trip it through llvm-as
+; RUN: llvm-dis -o - %t.o | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=DIS
; CHECK: <GLOBALVAL_SUMMARY_BLOCK
; CHECK-NEXT: <VERSION op0=9/>
; CHECK-NEXT: <FLAGS op0=0/>
+; The `VALUE_GUID` below represents the "_ZTV4Base" referenced by the instruction
+; that loads vtable pointers.
+; CHECK-NEXT: <VALUE_GUID op0=21 op1=1960855528937986108/>
; The `VALUE_GUID` below represents the "_ZN4Base4funcEv" referenced by the
; indirect call instruction.
-; CHECK-NEXT: <VALUE_GUID op0=17 op1=5459407273543877811/>
+; CHECK-NEXT: <VALUE_GUID op0=20 op1=5459407273543877811/>
+; NOTE vtables and functions from Derived class is dropped because
+; `-icp-max-num-vtables` and `-icp-max-prom` are both set to one.
; <PERMODULE_PROFILE> has the format [valueid, flags, instcount, funcflags,
; numrefs, rorefcnt, worefcnt,
+; m x valueid,
; n x (valueid, hotness+tailcall)]
-; CHECK-NEXT: <PERMODULE_PROFILE abbrevid=4 op0=0 op1=0 op2=4 op3=256 op4=0 op5=0 op6=0 op7=17 op8=3/>
+; CHECK-NEXT: <PERMODULE_PROFILE abbrevid=4 op0=0 op1=0 op2=4 op3=256 op4=1 op5=1 op6=0 op7=21 op8=20 op9=3/>
; CHECK-NEXT: </GLOBALVAL_SUMMARY_BLOCK>
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -31,7 +42,6 @@ define i32 @_Z4testP4Base(ptr %0) !prof !15 {
!llvm.module.flags = !{!1}
-
!1 = !{i32 1, !"ProfileSummary", !2}
!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
!3 = !{!"ProfileFormat", !"InstrProf"}
@@ -48,14 +58,17 @@ define i32 @_Z4testP4Base(ptr %0) !prof !15 {
!14 = !{i32 999999, i64 1, i32 2}
!15 = !{!"function_entry_count", i32 150}
-; 1960855528937986108 is the MD5 hash of _ZTV4Base
-!16 = !{!"VP", i32 2, i64 1600, i64 1960855528937986108, i64 1600}
-; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv
-!17 = !{!"VP", i32 0, i64 1600, i64 5459407273543877811, i64 1600}
+; 1960855528937986108 is the MD5 hash of _ZTV4Base, and
+; 13870436605473471591 is the MD5 hash of _ZTV7Derived
+!16 = !{!"VP", i32 2, i64 150, i64 1960855528937986108, i64 100, i64 13870436605473471591, i64 50}
+; 5459407273543877811 is the MD5 hash of _ZN4Base4funcEv, and
+; 6174874150489409711 is the MD5 hash of _ZN7Derived4funcEv
+!17 = !{!"VP", i32 0, i64 150, i64 5459407273543877811, i64 100, i64 6174874150489409711, i64 50}
; ModuleSummaryIndex stores <guid, global-value summary> map in std::map; so
; global value summares are printed out in the order that gv's guid increases.
; DIS: ^0 = module: (path: "{{.*}}", hash: (0, 0, 0, 0, 0))
-; DIS: ^1 = gv: (guid: 5459407273543877811)
-; DIS: ^2 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^1, hotness: hot))))) ; guid = 15857150948103218965
-; DIS: ^3 = blockcount: 0
+; DIS: ^1 = gv: (guid: 1960855528937986108)
+; DIS: ^2 = gv: (guid: 5459407273543877811)
+; DIS: ^3 = gv: (name: "_Z4testP4Base", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), insts: 4, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 1, mustBeUnreachable: 0), calls: ((callee: ^2, hotness: hot)), refs: (readonly ^1)))) ; guid = 15857150948103218965
+; DIS: ^4 = blockcount: 0
More information about the llvm-commits
mailing list