[llvm] r254110 - [PGO] Implement ValueProfiling Closure interfaces for runtime value profile data
Xinliang David Li via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 25 15:31:18 PST 2015
Author: davidxl
Date: Wed Nov 25 17:31:18 2015
New Revision: 254110
URL: http://llvm.org/viewvc/llvm-project?rev=254110&view=rev
Log:
[PGO] Implement ValueProfiling Closure interfaces for runtime value profile data
This is one of the many steps to commonize value profiling support between profile
runtime and compiler/llvm tools.
After this change, profiler runtime now can share the same C APIs to do VP
serialization/deseriazation with LLVM host tools (and produces value data
in identical format between indexed and raw profile).
It is not yet enabled in profiler runtime yet.
Also added a unit test case to test runtime profile data serialization/deserialization
interfaces implemented using common closure code.
Modified:
llvm/trunk/include/llvm/ProfileData/InstrProf.h
llvm/trunk/lib/ProfileData/InstrProf.cpp
llvm/trunk/unittests/ProfileData/InstrProfTest.cpp
Modified: llvm/trunk/include/llvm/ProfileData/InstrProf.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/ProfileData/InstrProf.h?rev=254110&r1=254109&r2=254110&view=diff
==============================================================================
--- llvm/trunk/include/llvm/ProfileData/InstrProf.h (original)
+++ llvm/trunk/include/llvm/ProfileData/InstrProf.h Wed Nov 25 17:31:18 2015
@@ -553,7 +553,52 @@ typedef struct ValueProfRecordClosure {
ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
} ValueProfRecordClosure;
-/// Return the \c ValueProfRecord header size including the padding bytes.
+/* A wrapper struct that represents value profile runtime data.
+ * Like InstrProfRecord class which is used by profiling host tools,
+ * ValueProfRuntimeRecord also implements the abstract intefaces defined in
+ * ValueProfRecordClosure so that the runtime data can be serialized using
+ * shared C implementation. In this structure, NumValueSites and Nodes
+ * members are the primary fields while other fields hold the derived
+ * information for fast implementation of closure interfaces.
+ */
+typedef struct ValueProfRuntimeRecord {
+ /* Number of sites for each value profile kind. */
+ uint16_t *NumValueSites;
+ /* An array of linked-list headers. The size of of the array is the
+ * total number of value profile sites : sum(NumValueSites[*])). Each
+ * linked-list stores the values profiled for a value profile site. */
+ ValueProfNode **Nodes;
+
+ /* Total number of value profile kinds which have at least one
+ * value profile sites. */
+ uint32_t NumValueKinds;
+ /* An array recording the number of values tracked at each site.
+ * The size of the array is TotalNumValueSites.
+ */
+ uint8_t *SiteCountArray[IPVK_Last + 1];
+ ValueProfNode **NodesKind[IPVK_Last + 1];
+} ValueProfRuntimeRecord;
+
+/* Initialize the record for runtime value profile data. */
+void initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
+ uint16_t *NumValueSites,
+ ValueProfNode **Nodes);
+
+/* Release memory allocated for the runtime record. */
+void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord);
+
+/* Return the size of ValueProfData structure that can be used to store
+ the value profile data collected at runtime. */
+uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record);
+
+/* Return a ValueProfData instance that stores the data collected at runtime. */
+ValueProfData *
+serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record);
+
+
+/*! \brief Return the \c ValueProfRecord header size including the
+ * padding bytes.
+ */
inline uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
sizeof(uint8_t) * NumValueSites;
@@ -562,21 +607,24 @@ inline uint32_t getValueProfRecordHeader
return Size;
}
-/// Return the total size of the value profile record including the
-/// header and the value data.
+/*! \brief Return the total size of the value profile record including the
+ * header and the value data.
+ */
inline uint32_t getValueProfRecordSize(uint32_t NumValueSites,
uint32_t NumValueData) {
return getValueProfRecordHeaderSize(NumValueSites) +
sizeof(InstrProfValueData) * NumValueData;
}
-/// Return the pointer to the start of value data array.
+/*! \brief Return the pointer to the start of value data array.
+ */
inline InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
This->NumValueSites));
}
-/// Return the total number of value data for \c This record.
+/*! \brief Return the total number of value data for \c This record.
+ */
inline uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
uint32_t NumValueData = 0;
uint32_t I;
@@ -585,7 +633,8 @@ inline uint32_t getValueProfRecordNumVal
return NumValueData;
}
-/// Use this method to advance to the next \c This \c ValueProfRecord.
+/* \brief Use this method to advance to the next \c This \c ValueProfRecord.
+ */
inline ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
uint32_t NumValueData = getValueProfRecordNumValueData(This);
return (ValueProfRecord *)((char *)This +
@@ -593,11 +642,14 @@ inline ValueProfRecord *getValueProfReco
NumValueData));
}
-/// Return the first \c ValueProfRecord instance.
+/*! \brief Return the first \c ValueProfRecord instance.
+ */
inline ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
}
+
+
namespace IndexedInstrProf {
enum class HashT : uint32_t {
Modified: llvm/trunk/lib/ProfileData/InstrProf.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/ProfileData/InstrProf.cpp?rev=254110&r1=254109&r2=254110&view=diff
==============================================================================
--- llvm/trunk/lib/ProfileData/InstrProf.cpp (original)
+++ llvm/trunk/lib/ProfileData/InstrProf.cpp Wed Nov 25 17:31:18 2015
@@ -188,9 +188,10 @@ ValueProfData *serializeValueProfDataFro
return VPD;
}
-// C wrappers of InstrProfRecord member functions used in Closure.
-// These C wrappers are used as adaptors so that C++ code can be
-// invoked as callbacks.
+/*! \brief ValueProfRecordClosure Interface implementation for InstrProfRecord
+ * class. These C wrappers are used as adaptors so that C++ code can be
+ * invoked as callbacks.
+ */
uint32_t getNumValueKindsInstrProf(const void *Record) {
return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
}
@@ -262,6 +263,120 @@ ValueProfData::serializeFrom(const Instr
return VPD;
}
+/* The value profiler runtime library stores the value profile data
+ * for a given function in NumValueSites and Nodes. This is the
+ * method to initialize the RuntimeRecord with the runtime data to
+ * pre-compute the information needed to efficiently implement
+ * ValueProfRecordClosure's callback interfaces.
+ */
+void initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord,
+ uint16_t *NumValueSites,
+ ValueProfNode **Nodes) {
+ unsigned I, J, S = 0, NumValueKinds = 0;
+ RuntimeRecord->NumValueSites = NumValueSites;
+ RuntimeRecord->Nodes = Nodes;
+ for (I = 0; I <= IPVK_Last; I++) {
+ uint16_t N = NumValueSites[I];
+ if (!N) {
+ RuntimeRecord->SiteCountArray[I] = 0;
+ continue;
+ }
+ NumValueKinds++;
+ RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1);
+ RuntimeRecord->NodesKind[I] = &RuntimeRecord->Nodes[S];
+ for (J = 0; J < N; J++) {
+ uint8_t C = 0;
+ ValueProfNode *Site = RuntimeRecord->Nodes[S + J];
+ while (Site) {
+ C++;
+ Site = Site->Next;
+ }
+ if (C > UCHAR_MAX)
+ C = UCHAR_MAX;
+ RuntimeRecord->SiteCountArray[I][J] = C;
+ }
+ S += N;
+ }
+ RuntimeRecord->NumValueKinds = NumValueKinds;
+}
+
+void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) {
+ unsigned I;
+ for (I = 0; I <= IPVK_Last; I++) {
+ if (RuntimeRecord->SiteCountArray[I])
+ free(RuntimeRecord->SiteCountArray[I]);
+ }
+}
+
+/* ValueProfRecordClosure Interface implementation for
+ * ValueProfDataRuntimeRecord. */
+uint32_t getNumValueKindsRT(const void *R) {
+ return ((const ValueProfRuntimeRecord *)R)->NumValueKinds;
+}
+
+uint32_t getNumValueSitesRT(const void *R, uint32_t VK) {
+ return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK];
+}
+
+uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) {
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ return Record->SiteCountArray[VK][S];
+}
+
+uint32_t getNumValueDataRT(const void *R, uint32_t VK) {
+ unsigned I, S = 0;
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ if (Record->SiteCountArray[VK] == 0)
+ return 0;
+ for (I = 0; I < Record->NumValueSites[VK]; I++)
+ S += Record->SiteCountArray[VK][I];
+ return S;
+}
+
+void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK,
+ uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) {
+ unsigned I, N = 0;
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ N = getNumValueDataForSiteRT(R, VK, S);
+ ValueProfNode *VNode = Record->NodesKind[VK][S];
+ for (I = 0; I < N; I++) {
+ Dst[I] = VNode->VData;
+ VNode = VNode->Next;
+ }
+}
+
+ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) {
+ return (ValueProfData *)calloc(TotalSizeInBytes, 1);
+}
+
+static ValueProfRecordClosure RTRecordClosure = {0,
+ getNumValueKindsRT,
+ getNumValueSitesRT,
+ getNumValueDataRT,
+ getNumValueDataForSiteRT,
+ 0,
+ getValueForSiteRT,
+ allocValueProfDataRT};
+
+/* Return the size of ValueProfData structure to store data
+ * recorded in the runtime record.
+ */
+uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) {
+ RTRecordClosure.Record = Record;
+ return getValueProfDataSize(&RTRecordClosure);
+}
+
+/* Return a ValueProfData instance that stores the data collected
+ from runtime. */
+ValueProfData *
+serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record) {
+ RTRecordClosure.Record = Record;
+ return serializeValueProfDataFrom(&RTRecordClosure);
+}
+
+
+
+
void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
InstrProfRecord::ValueMapType *VMap) {
Record.reserveSites(Kind, NumValueSites);
@@ -273,6 +388,7 @@ void ValueProfRecord::deserializeTo(Inst
ValueData += ValueDataCount;
}
}
+
// For writing/serializing, Old is the host endianness, and New is
// byte order intended on disk. For Reading/deserialization, Old
// is the on-disk source endianness, and New is the host endianness.
Modified: llvm/trunk/unittests/ProfileData/InstrProfTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/unittests/ProfileData/InstrProfTest.cpp?rev=254110&r1=254109&r2=254110&view=diff
==============================================================================
--- llvm/trunk/unittests/ProfileData/InstrProfTest.cpp (original)
+++ llvm/trunk/unittests/ProfileData/InstrProfTest.cpp Wed Nov 25 17:31:18 2015
@@ -382,6 +382,98 @@ TEST_F(InstrProfTest, get_icall_data_mer
ASSERT_EQ(Max, VD[0].Count);
}
+// Synthesize runtime value profile data.
+ValueProfNode Site1Values[5] = {{{uint64_t("callee1"), 400}, &Site1Values[1]},
+ {{uint64_t("callee2"), 1000}, &Site1Values[2]},
+ {{uint64_t("callee3"), 500}, &Site1Values[3]},
+ {{uint64_t("callee4"), 300}, &Site1Values[4]},
+ {{uint64_t("callee5"), 100}, 0}};
+
+ValueProfNode Site2Values[4] = {{{uint64_t("callee5"), 800}, &Site2Values[1]},
+ {{uint64_t("callee3"), 1000}, &Site2Values[2]},
+ {{uint64_t("callee2"), 2500}, &Site2Values[3]},
+ {{uint64_t("callee1"), 1300}, 0}};
+
+ValueProfNode Site3Values[3] = {{{uint64_t("callee6"), 800}, &Site3Values[1]},
+ {{uint64_t("callee3"), 1000}, &Site3Values[2]},
+ {{uint64_t("callee4"), 5500}, 0}};
+
+ValueProfNode Site4Values[2] = {{{uint64_t("callee2"), 1800}, &Site4Values[1]},
+ {{uint64_t("callee3"), 2000}, 0}};
+
+static ValueProfNode *ValueProfNodes[5] = {&Site1Values[0], &Site2Values[0],
+ &Site3Values[0], &Site4Values[0], 0};
+static uint16_t NumValueSites[IPVK_Last + 1] = {5};
+TEST_F(InstrProfTest, runtime_value_prof_data_read_write) {
+ ValueProfRuntimeRecord RTRecord;
+ initializeValueProfRuntimeRecord(&RTRecord, &NumValueSites[0],
+ &ValueProfNodes[0]);
+
+ ValueProfData *VPData = serializeValueProfDataFromRT(&RTRecord);
+
+ InstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
+
+ VPData->deserializeTo(Record, 0);
+
+ // Now read data from Record and sanity check the data
+ ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget));
+ ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
+ ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
+ ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
+ ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
+ ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4));
+
+ auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
+ return VD1.Count > VD2.Count;
+ };
+ std::unique_ptr<InstrProfValueData[]> VD_0(
+ Record.getValueForSite(IPVK_IndirectCallTarget, 0));
+ std::sort(&VD_0[0], &VD_0[5], Cmp);
+ ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2"));
+ ASSERT_EQ(1000U, VD_0[0].Count);
+ ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3"));
+ ASSERT_EQ(500U, VD_0[1].Count);
+ ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1"));
+ ASSERT_EQ(400U, VD_0[2].Count);
+ ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4"));
+ ASSERT_EQ(300U, VD_0[3].Count);
+ ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5"));
+ ASSERT_EQ(100U, VD_0[4].Count);
+
+ std::unique_ptr<InstrProfValueData[]> VD_1(
+ Record.getValueForSite(IPVK_IndirectCallTarget, 1));
+ std::sort(&VD_1[0], &VD_1[4], Cmp);
+ ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2"));
+ ASSERT_EQ(2500U, VD_1[0].Count);
+ ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1"));
+ ASSERT_EQ(1300U, VD_1[1].Count);
+ ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3"));
+ ASSERT_EQ(1000U, VD_1[2].Count);
+ ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5"));
+ ASSERT_EQ(800U, VD_1[3].Count);
+
+ std::unique_ptr<InstrProfValueData[]> VD_2(
+ Record.getValueForSite(IPVK_IndirectCallTarget, 2));
+ std::sort(&VD_2[0], &VD_2[3], Cmp);
+ ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4"));
+ ASSERT_EQ(5500U, VD_2[0].Count);
+ ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3"));
+ ASSERT_EQ(1000U, VD_2[1].Count);
+ ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6"));
+ ASSERT_EQ(800U, VD_2[2].Count);
+
+ std::unique_ptr<InstrProfValueData[]> VD_3(
+ Record.getValueForSite(IPVK_IndirectCallTarget, 3));
+ std::sort(&VD_3[0], &VD_3[2], Cmp);
+ ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3"));
+ ASSERT_EQ(2000U, VD_3[0].Count);
+ ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2"));
+ ASSERT_EQ(1800U, VD_3[1].Count);
+
+ finalizeValueProfRuntimeRecord(&RTRecord);
+ free(VPData);
+}
+
TEST_F(InstrProfTest, get_max_function_count) {
InstrProfRecord Record1("foo", 0x1234, {1ULL << 31, 2});
InstrProfRecord Record2("bar", 0, {1ULL << 63});
More information about the llvm-commits
mailing list